From 89d5d1cafb23280bda3f303442387e71353c9e49 Mon Sep 17 00:00:00 2001 From: Ryuta Kamizono Date: Tue, 4 Aug 2015 04:16:54 +0900 Subject: Add a native JSON data type support in MySQL As of MySQL 5.7.8, MySQL supports a native JSON data type. Example: create_table :json_data_type do |t| t.json :settings end --- activerecord/CHANGELOG.md | 10 ++ .../connection_adapters/abstract_adapter.rb | 5 + .../connection_adapters/abstract_mysql_adapter.rb | 39 +++-- .../connection_adapters/mysql2_adapter.rb | 4 + .../connection_adapters/postgresql/oid.rb | 1 - .../connection_adapters/postgresql/oid/json.rb | 35 ----- .../connection_adapters/postgresql/oid/jsonb.rb | 2 +- .../connection_adapters/postgresql_adapter.rb | 7 +- activerecord/lib/active_record/type.rb | 2 + activerecord/lib/active_record/type/json.rb | 31 ++++ .../test/cases/adapters/mysql2/json_test.rb | 172 +++++++++++++++++++++ 11 files changed, 258 insertions(+), 50 deletions(-) delete mode 100644 activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb create mode 100644 activerecord/lib/active_record/type/json.rb create mode 100644 activerecord/test/cases/adapters/mysql2/json_test.rb diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index bd86dfb88e..baaa38466b 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,13 @@ +* Add a native JSON data type support in MySQL. + + Example: + + create_table :json_data_type do |t| + t.json :settings + end + + *Ryuta Kamizono* + * Descriptive error message when fixtures contain a missing column. Closes #21201. diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 56227ddd80..ed14c781c6 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -266,6 +266,11 @@ module ActiveRecord false end + # Does this adapter support json data type? + def supports_json? + false + end + # This is meant to be implemented by the adapters that support extensions def disable_extension(name) end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index af156c9c78..96ea866580 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -10,6 +10,10 @@ module ActiveRecord options[:auto_increment] = true if type == :bigint super end + + def json(*args, **options) + args.each { |name| column(name, :json, options) } + end end class ColumnDefinition < ActiveRecord::ConnectionAdapters::ColumnDefinition @@ -242,17 +246,19 @@ module ActiveRecord QUOTED_TRUE, QUOTED_FALSE = '1', '0' NATIVE_DATABASE_TYPES = { - :primary_key => "int(11) auto_increment PRIMARY KEY", - :string => { :name => "varchar", :limit => 255 }, - :text => { :name => "text" }, - :integer => { :name => "int", :limit => 4 }, - :float => { :name => "float" }, - :decimal => { :name => "decimal" }, - :datetime => { :name => "datetime" }, - :time => { :name => "time" }, - :date => { :name => "date" }, - :binary => { :name => "blob" }, - :boolean => { :name => "tinyint", :limit => 1 } + primary_key: "int(11) auto_increment PRIMARY KEY", + string: { name: "varchar", limit: 255 }, + text: { name: "text" }, + integer: { name: "int", limit: 4 }, + float: { name: "float" }, + decimal: { name: "decimal" }, + datetime: { name: "datetime" }, + time: { name: "time" }, + date: { name: "date" }, + binary: { name: "blob" }, + boolean: { name: "tinyint", limit: 1 }, + bigint: { name: "bigint" }, + json: { name: "json" }, } INDEX_TYPES = [:fulltext, :spatial] @@ -790,6 +796,7 @@ module ActiveRecord m.register_type %r(longblob)i, Type::Binary.new(limit: 2**32 - 1) m.register_type %r(^float)i, Type::Float.new(limit: 24) m.register_type %r(^double)i, Type::Float.new(limit: 53) + m.register_type %r(^json)i, MysqlJson.new register_integer_type m, %r(^bigint)i, limit: 8 register_integer_type m, %r(^int)i, limit: 4 @@ -1043,6 +1050,14 @@ module ActiveRecord end end + class MysqlJson < Type::Json # :nodoc: + def changed_in_place?(raw_old_value, new_value) + # Normalization is required because MySQL JSON data format includes + # the space between the elements. + super(serialize(deserialize(raw_old_value)), new_value) + end + end + class MysqlString < Type::String # :nodoc: def serialize(value) case value @@ -1063,6 +1078,8 @@ module ActiveRecord end end + ActiveRecord::Type.register(:json, MysqlJson, adapter: :mysql) + ActiveRecord::Type.register(:json, MysqlJson, adapter: :mysql2) ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql) ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql2) end diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb index b7db57c9fe..ff43c7ec42 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb @@ -41,6 +41,10 @@ module ActiveRecord true end + def supports_json? + version >= '5.7.8' + end + # HELPER METHODS =========================================== def each_hash(result) # :nodoc: diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb index 68752cdd80..37226831dc 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb @@ -8,7 +8,6 @@ require 'active_record/connection_adapters/postgresql/oid/decimal' require 'active_record/connection_adapters/postgresql/oid/enum' require 'active_record/connection_adapters/postgresql/oid/hstore' require 'active_record/connection_adapters/postgresql/oid/inet' -require 'active_record/connection_adapters/postgresql/oid/json' require 'active_record/connection_adapters/postgresql/oid/jsonb' require 'active_record/connection_adapters/postgresql/oid/money' require 'active_record/connection_adapters/postgresql/oid/point' diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb deleted file mode 100644 index 8e1256baad..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb +++ /dev/null @@ -1,35 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - class Json < Type::Value # :nodoc: - include Type::Helpers::Mutable - - def type - :json - end - - def deserialize(value) - if value.is_a?(::String) - ::ActiveSupport::JSON.decode(value) rescue nil - else - value - end - end - - def serialize(value) - if value.is_a?(::Array) || value.is_a?(::Hash) - ::ActiveSupport::JSON.encode(value) - else - value - end - end - - def accessor - ActiveRecord::Store::StringKeyedHashAccessor - end - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb index 87391b5dc7..1f6d63582c 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb @@ -2,7 +2,7 @@ module ActiveRecord module ConnectionAdapters module PostgreSQL module OID # :nodoc: - class Jsonb < Json # :nodoc: + class Jsonb < Type::Json # :nodoc: def type :jsonb end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index 2c43c46a3d..861edbf3a2 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -201,6 +201,10 @@ module ActiveRecord true end + def supports_json? + postgresql_version >= 90200 + end + def index_algorithms { concurrently: 'CONCURRENTLY' } end @@ -478,7 +482,7 @@ module ActiveRecord m.register_type 'bytea', OID::Bytea.new m.register_type 'point', OID::Point.new m.register_type 'hstore', OID::Hstore.new - m.register_type 'json', OID::Json.new + m.register_type 'json', Type::Json.new m.register_type 'jsonb', OID::Jsonb.new m.register_type 'cidr', OID::Cidr.new m.register_type 'inet', OID::Inet.new @@ -834,7 +838,6 @@ module ActiveRecord ActiveRecord::Type.register(:enum, OID::Enum, adapter: :postgresql) ActiveRecord::Type.register(:hstore, OID::Hstore, adapter: :postgresql) ActiveRecord::Type.register(:inet, OID::Inet, adapter: :postgresql) - ActiveRecord::Type.register(:json, OID::Json, adapter: :postgresql) ActiveRecord::Type.register(:jsonb, OID::Jsonb, adapter: :postgresql) ActiveRecord::Type.register(:money, OID::Money, adapter: :postgresql) ActiveRecord::Type.register(:point, OID::Point, adapter: :postgresql) diff --git a/activerecord/lib/active_record/type.rb b/activerecord/lib/active_record/type.rb index 2c0cda69d0..8f56a37f3c 100644 --- a/activerecord/lib/active_record/type.rb +++ b/activerecord/lib/active_record/type.rb @@ -10,6 +10,7 @@ require 'active_record/type/decimal' require 'active_record/type/decimal_without_scale' require 'active_record/type/float' require 'active_record/type/integer' +require 'active_record/type/json' require 'active_record/type/serialized' require 'active_record/type/string' require 'active_record/type/text' @@ -59,6 +60,7 @@ module ActiveRecord register(:decimal, Type::Decimal, override: false) register(:float, Type::Float, override: false) register(:integer, Type::Integer, override: false) + register(:json, Type::Json, override: false) register(:string, Type::String, override: false) register(:text, Type::Text, override: false) register(:time, Type::Time, override: false) diff --git a/activerecord/lib/active_record/type/json.rb b/activerecord/lib/active_record/type/json.rb new file mode 100644 index 0000000000..1728bd3a8e --- /dev/null +++ b/activerecord/lib/active_record/type/json.rb @@ -0,0 +1,31 @@ +module ActiveRecord + module Type + class Json < Type::Value # :nodoc: + include Type::Helpers::Mutable + + def type + :json + end + + def deserialize(value) + if value.is_a?(::String) + ::ActiveSupport::JSON.decode(value) rescue nil + else + value + end + end + + def serialize(value) + if value.is_a?(::Array) || value.is_a?(::Hash) + ::ActiveSupport::JSON.encode(value) + else + value + end + end + + def accessor + ActiveRecord::Store::StringKeyedHashAccessor + end + end + end +end diff --git a/activerecord/test/cases/adapters/mysql2/json_test.rb b/activerecord/test/cases/adapters/mysql2/json_test.rb new file mode 100644 index 0000000000..c8c933af5e --- /dev/null +++ b/activerecord/test/cases/adapters/mysql2/json_test.rb @@ -0,0 +1,172 @@ +require 'cases/helper' +require 'support/schema_dumping_helper' + +if ActiveRecord::Base.connection.supports_json? +class Mysql2JSONTest < ActiveRecord::Mysql2TestCase + include SchemaDumpingHelper + self.use_transactional_tests = false + + class JsonDataType < ActiveRecord::Base + self.table_name = 'json_data_type' + + store_accessor :settings, :resolution + end + + def setup + @connection = ActiveRecord::Base.connection + begin + @connection.create_table('json_data_type') do |t| + t.json 'payload' + t.json 'settings' + end + end + end + + def teardown + @connection.drop_table :json_data_type, if_exists: true + JsonDataType.reset_column_information + end + + def test_column + column = JsonDataType.columns_hash["payload"] + assert_equal :json, column.type + assert_equal 'json', column.sql_type + + type = JsonDataType.type_for_attribute("payload") + assert_not type.binary? + end + + def test_change_table_supports_json + @connection.change_table('json_data_type') do |t| + t.json 'users' + end + JsonDataType.reset_column_information + column = JsonDataType.columns_hash['users'] + assert_equal :json, column.type + end + + def test_schema_dumping + output = dump_table_schema("json_data_type") + assert_match(/t\.json\s+"settings"/, output) + end + + def test_cast_value_on_write + x = JsonDataType.new payload: {"string" => "foo", :symbol => :bar} + assert_equal({"string" => "foo", :symbol => :bar}, x.payload_before_type_cast) + assert_equal({"string" => "foo", "symbol" => "bar"}, x.payload) + x.save + assert_equal({"string" => "foo", "symbol" => "bar"}, x.reload.payload) + end + + def test_type_cast_json + type = JsonDataType.type_for_attribute("payload") + + data = "{\"a_key\":\"a_value\"}" + hash = type.deserialize(data) + assert_equal({'a_key' => 'a_value'}, hash) + assert_equal({'a_key' => 'a_value'}, type.deserialize(data)) + + assert_equal({}, type.deserialize("{}")) + assert_equal({'key'=>nil}, type.deserialize('{"key": null}')) + assert_equal({'c'=>'}','"a"'=>'b "a b'}, type.deserialize(%q({"c":"}", "\"a\"":"b \"a b"}))) + end + + def test_rewrite + @connection.execute "insert into json_data_type (payload) VALUES ('{\"k\":\"v\"}')" + x = JsonDataType.first + x.payload = { '"a\'' => 'b' } + assert x.save! + end + + def test_select + @connection.execute "insert into json_data_type (payload) VALUES ('{\"k\":\"v\"}')" + x = JsonDataType.first + assert_equal({'k' => 'v'}, x.payload) + end + + def test_select_multikey + @connection.execute %q|insert into json_data_type (payload) VALUES ('{"k1":"v1", "k2":"v2", "k3":[1,2,3]}')| + x = JsonDataType.first + assert_equal({'k1' => 'v1', 'k2' => 'v2', 'k3' => [1,2,3]}, x.payload) + end + + def test_null_json + @connection.execute %q|insert into json_data_type (payload) VALUES(null)| + x = JsonDataType.first + assert_equal(nil, x.payload) + end + + def test_select_array_json_value + @connection.execute %q|insert into json_data_type (payload) VALUES ('["v0",{"k1":"v1"}]')| + x = JsonDataType.first + assert_equal(['v0', {'k1' => 'v1'}], x.payload) + end + + def test_rewrite_array_json_value + @connection.execute %q|insert into json_data_type (payload) VALUES ('["v0",{"k1":"v1"}]')| + x = JsonDataType.first + x.payload = ['v1', {'k2' => 'v2'}, 'v3'] + assert x.save! + end + + def test_with_store_accessors + x = JsonDataType.new(resolution: "320×480") + assert_equal "320×480", x.resolution + + x.save! + x = JsonDataType.first + assert_equal "320×480", x.resolution + + x.resolution = "640×1136" + x.save! + + x = JsonDataType.first + assert_equal "640×1136", x.resolution + end + + def test_duplication_with_store_accessors + x = JsonDataType.new(resolution: "320×480") + assert_equal "320×480", x.resolution + + y = x.dup + assert_equal "320×480", y.resolution + end + + def test_yaml_round_trip_with_store_accessors + x = JsonDataType.new(resolution: "320×480") + assert_equal "320×480", x.resolution + + y = YAML.load(YAML.dump(x)) + assert_equal "320×480", y.resolution + end + + def test_changes_in_place + json = JsonDataType.new + assert_not json.changed? + + json.payload = { 'one' => 'two' } + assert json.changed? + assert json.payload_changed? + + json.save! + assert_not json.changed? + + json.payload['three'] = 'four' + assert json.payload_changed? + + json.save! + json.reload + + assert_equal({ 'one' => 'two', 'three' => 'four' }, json.payload) + assert_not json.changed? + end + + def test_assigning_invalid_json + json = JsonDataType.new + + json.payload = 'foo' + + assert_nil json.payload + end +end +end -- cgit v1.2.3