From 4544d2bc90bea93c38bb21d912dba00f51cf620f Mon Sep 17 00:00:00 2001 From: Dan McClain Date: Sun, 19 Aug 2012 18:02:34 -0400 Subject: Moves column dump specific code to a module included in AbstractAdapter Having column related schema dumper code in the AbstractAdapter. The code remains the same, but by placing it in the AbstractAdapter, we can then overwrite it with Adapter specific methods that will help with Adapter specific data types. The goal of moving this code here is to create a new migration key for PostgreSQL's array type. Since any datatype can be an array, the goal is to have ':array => true' as a migration option, turning the datatype into an array. I've implemented this in postgres_ext, the syntax is shown here: https://github.com/dockyard/postgres_ext#arrays Adds array migration support Adds array_test.rb outlining the test cases for array data type Adds pg_array_parser to Gemfile for testing Adds pg_array_parser to postgresql_adapter (unused in this commit) Adds schema dump support for arrays Adds postgres array type casting support Updates changelog, adds note for inet and cidr support, which I forgot to add before Removing debugger, Adds pg_array_parser to JRuby platform Removes pg_array_parser requirement, creates ArrayParser module used by PostgreSQLAdapter --- .../connection_adapters/abstract/schema_dumper.rb | 56 +++++++++++++ .../connection_adapters/abstract_adapter.rb | 2 + .../connection_adapters/postgresql/array_parser.rb | 97 ++++++++++++++++++++++ .../connection_adapters/postgresql/cast.rb | 28 +++++++ .../connection_adapters/postgresql/oid.rb | 15 ++++ .../connection_adapters/postgresql/quoting.rb | 29 +++++-- .../connection_adapters/postgresql_adapter.rb | 75 +++++++++++++++-- activerecord/lib/active_record/schema_dumper.rb | 31 +------ 8 files changed, 289 insertions(+), 44 deletions(-) create mode 100644 activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb create mode 100644 activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb (limited to 'activerecord/lib/active_record') diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb new file mode 100644 index 0000000000..9d6111b51e --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -0,0 +1,56 @@ +module ActiveRecord + module ConnectionAdapters # :nodoc: + # The goal of this module is to move Adapter specific column + # definitions to the Adapter instead of having it in the schema + # dumper itself. This code represents the normal case. + # We can then redefine how certain data types may be handled in the schema dumper on the + # Adapter level by over-writing this code inside the database spececific adapters + module ColumnDumper + def column_spec(column, types) + spec = prepare_column_options(column, types) + (spec.keys - [:name, :type]).each{ |k| spec[k].insert(0, "#{k.to_s}: ")} + spec + end + + # This can be overridden on a Adapter level basis to support other + # extended datatypes (Example: Adding an array option in the + # PostgreSQLAdapter) + def prepare_column_options(column, types) + spec = {} + spec[:name] = column.name.inspect + + # AR has an optimization which handles zero-scale decimals as integers. This + # code ensures that the dumper still dumps the column as a decimal. + spec[:type] = if column.type == :integer && /^(numeric|decimal)/ =~ column.sql_type + 'decimal' + else + column.type.to_s + end + spec[:limit] = column.limit.inspect if column.limit != types[column.type][:limit] && spec[:type] != 'decimal' + spec[:precision] = column.precision.inspect if column.precision + spec[:scale] = column.scale.inspect if column.scale + spec[:null] = 'false' unless column.null + spec[:default] = default_string(column.default) if column.has_default? + spec + end + + # Lists the valid migration options + def migration_keys + [:name, :limit, :precision, :scale, :default, :null] + end + + private + + def default_string(value) + case value + when BigDecimal + value.to_s + when Date, DateTime, Time + "'#{value.to_s(:db)}'" + else + value.inspect + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 27700e4fd2..d5c7caad2e 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -3,6 +3,7 @@ require 'bigdecimal' require 'bigdecimal/util' require 'active_support/core_ext/benchmark' require 'active_record/connection_adapters/schema_cache' +require 'active_record/connection_adapters/abstract/schema_dumper' require 'monitor' module ActiveRecord @@ -52,6 +53,7 @@ module ActiveRecord include QueryCache include ActiveSupport::Callbacks include MonitorMixin + include ColumnDumper define_callbacks :checkout, :checkin diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb b/activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb new file mode 100644 index 0000000000..b7d24f2bb3 --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb @@ -0,0 +1,97 @@ +module ActiveRecord + module ConnectionAdapters + class PostgreSQLColumn < Column + module ArrayParser + private + # Loads pg_array_parser if available. String parsing can be + # performed quicker by a native extension, which will not create + # a large amount of Ruby objects that will need to be garbage + # collected. pg_array_parser has a C and Java extension + begin + require 'pg_array_parser' + include PgArrayParser + rescue LoadError + def parse_pg_array(string) + parse_data(string, 0) + end + end + + def parse_data(string, index) + local_index = index + array = [] + while(local_index < string.length) + case string[local_index] + when '{' + local_index,array = parse_array_contents(array, string, local_index + 1) + when '}' + return array + end + local_index += 1 + end + + array + end + + def parse_array_contents(array, string, index) + is_escaping = false + is_quoted = false + was_quoted = false + current_item = '' + + local_index = index + while local_index + token = string[local_index] + if is_escaping + current_item << token + is_escaping = false + else + if is_quoted + case token + when '"' + is_quoted = false + was_quoted = true + when "\\" + is_escaping = true + else + current_item << token + end + else + case token + when "\\" + is_escaping = true + when ',' + add_item_to_array(array, current_item, was_quoted) + current_item = '' + was_quoted = false + when '"' + is_quoted = true + when '{' + internal_items = [] + local_index,internal_items = parse_array_contents(internal_items, string, local_index + 1) + array.push(internal_items) + when '}' + add_item_to_array(array, current_item, was_quoted) + return local_index,array + else + current_item << token + end + end + end + + local_index += 1 + end + return local_index,array + end + + def add_item_to_array(array, current_item, quoted) + if current_item.length == 0 + elsif !quoted && current_item == 'NULL' + array.push nil + else + array.push current_item + end + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/cast.rb b/activerecord/lib/active_record/connection_adapters/postgresql/cast.rb index b59195f98a..62d091357d 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/cast.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/cast.rb @@ -45,6 +45,21 @@ module ActiveRecord end end + def array_to_string(value, column, adapter, should_be_quoted = false) + casted_values = value.map do |val| + if String === val + if val == "NULL" + "\"#{val}\"" + else + quote_and_escape(adapter.type_cast(val, column, true)) + end + else + adapter.type_cast(val, column, true) + end + end + "{#{casted_values.join(',')}}" + end + def string_to_json(string) if String === string ActiveSupport::JSON.decode(string) @@ -71,6 +86,10 @@ module ActiveRecord end end + def string_to_array(string, oid) + parse_pg_array(string).map{|val| oid.type_cast val} + end + private HstorePair = begin @@ -90,6 +109,15 @@ module ActiveRecord end end end + + def quote_and_escape(value) + case value + when "NULL" + value + else + "\"#{value.gsub(/"/,"\\\"")}\"" + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb index b8e7687b21..52344f61c0 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb @@ -63,6 +63,21 @@ module ActiveRecord end end + class Array < Type + attr_reader :subtype + def initialize(subtype) + @subtype = subtype + end + + def type_cast(value) + if String === value + ConnectionAdapters::PostgreSQLColumn.string_to_array value, @subtype + else + value + end + end + end + class Integer < Type def type_cast(value) return if value.nil? diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb index 85721601a9..37d43d891d 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb @@ -19,6 +19,12 @@ module ActiveRecord return super unless column case value + when Array + if column.array + "'#{PostgreSQLColumn.array_to_string(value, column, self)}'" + else + super + end when Hash case column.sql_type when 'hstore' then super(PostgreSQLColumn.hstore_to_string(value), column) @@ -59,24 +65,35 @@ module ActiveRecord end end - def type_cast(value, column) - return super unless column + def type_cast(value, column, array_member = false) + return super(value, column) unless column case value + when NilClass + if column.array && array_member + 'NULL' + elsif column.array + value + else + super(value, column) + end + when Array + return super(value, column) unless column.array + PostgreSQLColumn.array_to_string(value, column, self) when String - return super unless 'bytea' == column.sql_type + return super(value, column) unless 'bytea' == column.sql_type { :value => value, :format => 1 } when Hash case column.sql_type when 'hstore' then PostgreSQLColumn.hstore_to_string(value) when 'json' then PostgreSQLColumn.json_to_string(value) - else super + else super(value, column) end when IPAddr - return super unless ['inet','cidr'].includes? column.sql_type + return super(value, column) unless ['inet','cidr'].includes? column.sql_type PostgreSQLColumn.cidr_to_string(value) else - super + super(value, column) end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index d1751d70c6..1a727f9aa6 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -2,6 +2,7 @@ require 'active_record/connection_adapters/abstract_adapter' require 'active_record/connection_adapters/statement_pool' require 'active_record/connection_adapters/postgresql/oid' require 'active_record/connection_adapters/postgresql/cast' +require 'active_record/connection_adapters/postgresql/array_parser' require 'active_record/connection_adapters/postgresql/quoting' require 'active_record/connection_adapters/postgresql/schema_statements' require 'active_record/connection_adapters/postgresql/database_statements' @@ -41,16 +42,23 @@ module ActiveRecord module ConnectionAdapters # PostgreSQL-specific extensions to column definitions in a table. class PostgreSQLColumn < Column #:nodoc: + attr_accessor :array # Instantiates a new PostgreSQL column definition in a table. def initialize(name, default, oid_type, sql_type = nil, null = true) @oid_type = oid_type - super(name, self.class.extract_value_from_default(default), sql_type, null) + if sql_type =~ /\[\]$/ + @array = true + super(name, self.class.extract_value_from_default(default), sql_type[0..sql_type.length - 3], null) + else + @array = false + super(name, self.class.extract_value_from_default(default), sql_type, null) + end end # :stopdoc: class << self include ConnectionAdapters::PostgreSQLColumn::Cast - + include ConnectionAdapters::PostgreSQLColumn::ArrayParser attr_accessor :money_precision end # :startdoc: @@ -243,6 +251,10 @@ module ActiveRecord # In addition, default connection parameters of libpq can be set per environment variables. # See http://www.postgresql.org/docs/9.1/static/libpq-envars.html . class PostgreSQLAdapter < AbstractAdapter + class ColumnDefinition < ActiveRecord::ConnectionAdapters::ColumnDefinition + attr_accessor :array + end + class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition def xml(*args) options = args.extract_options! @@ -277,6 +289,23 @@ module ActiveRecord def json(name, options = {}) column(name, 'json', options) end + + def column(name, type = nil, options = {}) + super + column = self[name] + column.array = options[:array] + + self + end + + private + + def new_column_definition(base, name, type) + definition = ColumnDefinition.new base, name, type + @columns << definition + @columns_hash[name] = definition + definition + end end ADAPTER_NAME = 'PostgreSQL' @@ -314,6 +343,19 @@ module ActiveRecord ADAPTER_NAME end + # Adds `:array` option to the default set provided by the + # AbstractAdapter + def prepare_column_options(column, types) + spec = super + spec[:array] = 'true' if column.respond_to?(:array) && column.array + spec + end + + # Adds `:array` as a valid migration key + def migration_keys + super + [:array] + end + # Returns +true+, since this connection adapter supports prepared statement # caching. def supports_statement_cache? @@ -494,6 +536,13 @@ module ActiveRecord @table_alias_length ||= query('SHOW max_identifier_length', 'SCHEMA')[0][0].to_i end + def add_column_options!(sql, options) + if options[:array] || options[:column].try(:array) + sql << '[]' + end + super + end + # Set the authorized user for this session def session_auth=(user) clear_cache! @@ -548,7 +597,7 @@ module ActiveRecord private def initialize_type_map - result = execute('SELECT oid, typname, typelem, typdelim FROM pg_type', 'SCHEMA') + result = execute('SELECT oid, typname, typelem, typdelim, typinput FROM pg_type', 'SCHEMA') leaves, nodes = result.partition { |row| row['typelem'] == '0' } # populate the leaf nodes @@ -556,11 +605,19 @@ module ActiveRecord OID::TYPE_MAP[row['oid'].to_i] = OID::NAMES[row['typname']] end + arrays, nodes = nodes.partition { |row| row['typinput'] == 'array_in' } + # populate composite types nodes.find_all { |row| OID::TYPE_MAP.key? row['typelem'].to_i }.each do |row| vector = OID::Vector.new row['typdelim'], OID::TYPE_MAP[row['typelem'].to_i] OID::TYPE_MAP[row['oid'].to_i] = vector end + + # populate array types + arrays.find_all { |row| OID::TYPE_MAP.key? row['typelem'].to_i }.each do |row| + array = OID::Array.new OID::TYPE_MAP[row['typelem'].to_i] + OID::TYPE_MAP[row['oid'].to_i] = array + end end FEATURE_NOT_SUPPORTED = "0A000" # :nodoc: @@ -703,12 +760,12 @@ module ActiveRecord # - ::regclass is a function that gives the id for a table name def column_definitions(table_name) #:nodoc: exec_query(<<-end_sql, 'SCHEMA').rows - SELECT a.attname, format_type(a.atttypid, a.atttypmod), d.adsrc, a.attnotnull, a.atttypid, a.atttypmod - FROM pg_attribute a LEFT JOIN pg_attrdef d - ON a.attrelid = d.adrelid AND a.attnum = d.adnum - WHERE a.attrelid = '#{quote_table_name(table_name)}'::regclass - AND a.attnum > 0 AND NOT a.attisdropped - ORDER BY a.attnum + SELECT a.attname, format_type(a.atttypid, a.atttypmod), d.adsrc, a.attnotnull, a.atttypid, a.atttypmod + FROM pg_attribute a LEFT JOIN pg_attrdef d + ON a.attrelid = d.adrelid AND a.attnum = d.adnum + WHERE a.attrelid = '#{quote_table_name(table_name)}'::regclass + AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum end_sql end diff --git a/activerecord/lib/active_record/schema_dumper.rb b/activerecord/lib/active_record/schema_dumper.rb index 310b4c1459..36bde44e7c 100644 --- a/activerecord/lib/active_record/schema_dumper.rb +++ b/activerecord/lib/active_record/schema_dumper.rb @@ -107,27 +107,11 @@ HEADER column_specs = columns.map do |column| raise StandardError, "Unknown type '#{column.sql_type}' for column '#{column.name}'" if @types[column.type].nil? next if column.name == pk - spec = {} - spec[:name] = column.name.inspect - - # AR has an optimization which handles zero-scale decimals as integers. This - # code ensures that the dumper still dumps the column as a decimal. - spec[:type] = if column.type == :integer && /^(numeric|decimal)/ =~ column.sql_type - 'decimal' - else - column.type.to_s - end - spec[:limit] = column.limit.inspect if column.limit != @types[column.type][:limit] && spec[:type] != 'decimal' - spec[:precision] = column.precision.inspect if column.precision - spec[:scale] = column.scale.inspect if column.scale - spec[:null] = 'false' unless column.null - spec[:default] = default_string(column.default) if column.has_default? - (spec.keys - [:name, :type]).each{ |k| spec[k].insert(0, "#{k.to_s}: ")} - spec + @connection.column_spec(column, @types) end.compact # find all migration keys used in this table - keys = [:name, :limit, :precision, :scale, :default, :null] + keys = @connection.migration_keys # figure out the lengths for each column based on above keys lengths = keys.map { |key| @@ -170,17 +154,6 @@ HEADER stream end - def default_string(value) - case value - when BigDecimal - value.to_s - when Date, DateTime, Time - "'#{value.to_s(:db)}'" - else - value.inspect - end - end - def indexes(table, stream) if (indexes = @connection.indexes(table)).any? add_index_statements = indexes.map do |index| -- cgit v1.2.3