Merge pull request #21110 from kamipo/mysql_json_support

Add a native JSON data type support in MySQL

11 files changed, 258 insertions, 50 deletions

diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md
index bd86dfb88e..baaa38466b 100644
--- a/activerecord/CHANGELOG.md
+++ b/activerecord/CHANGELOG.md
@@ -1,3 +1,13 @@
+*   Add a native JSON data type support in MySQL.
+
+    Example:
+
+        create_table :json_data_type do |t|
+          t.json :settings
+        end
+
+    *Ryuta Kamizono*
+
 *   Descriptive error message when fixtures contain a missing column.
 
     Closes #21201.
diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb
index 56227ddd80..ed14c781c6 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb
@@ -266,6 +266,11 @@ module ActiveRecord
         false
       end
 
+      # Does this adapter support json data type?
+      def supports_json?
+        false
+      end
+
       # This is meant to be implemented by the adapters that support extensions
       def disable_extension(name)
       end
diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb
index af156c9c78..96ea866580 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb
@@ -10,6 +10,10 @@ module ActiveRecord
           options[:auto_increment] = true if type == :bigint
           super
         end
+
+        def json(*args, **options)
+          args.each { |name| column(name, :json, options) }
+        end
       end
 
       class ColumnDefinition < ActiveRecord::ConnectionAdapters::ColumnDefinition
@@ -242,17 +246,19 @@ module ActiveRecord
       QUOTED_TRUE, QUOTED_FALSE = '1', '0'
 
       NATIVE_DATABASE_TYPES = {
-        :primary_key => "int(11) auto_increment PRIMARY KEY",
-        :string      => { :name => "varchar", :limit => 255 },
-        :text        => { :name => "text" },
-        :integer     => { :name => "int", :limit => 4 },
-        :float       => { :name => "float" },
-        :decimal     => { :name => "decimal" },
-        :datetime    => { :name => "datetime" },
-        :time        => { :name => "time" },
-        :date        => { :name => "date" },
-        :binary      => { :name => "blob" },
-        :boolean     => { :name => "tinyint", :limit => 1 }
+        primary_key: "int(11) auto_increment PRIMARY KEY",
+        string:      { name: "varchar", limit: 255 },
+        text:        { name: "text" },
+        integer:     { name: "int", limit: 4 },
+        float:       { name: "float" },
+        decimal:     { name: "decimal" },
+        datetime:    { name: "datetime" },
+        time:        { name: "time" },
+        date:        { name: "date" },
+        binary:      { name: "blob" },
+        boolean:     { name: "tinyint", limit: 1 },
+        bigint:      { name: "bigint" },
+        json:        { name: "json" },
       }
 
       INDEX_TYPES  = [:fulltext, :spatial]
@@ -790,6 +796,7 @@ module ActiveRecord
         m.register_type %r(longblob)i,   Type::Binary.new(limit: 2**32 - 1)
         m.register_type %r(^float)i,     Type::Float.new(limit: 24)
         m.register_type %r(^double)i,    Type::Float.new(limit: 53)
+        m.register_type %r(^json)i,      MysqlJson.new
 
         register_integer_type m, %r(^bigint)i,    limit: 8
         register_integer_type m, %r(^int)i,       limit: 4
@@ -1043,6 +1050,14 @@ module ActiveRecord
         end
       end
 
+      class MysqlJson < Type::Json # :nodoc:
+        def changed_in_place?(raw_old_value, new_value)
+          # Normalization is required because MySQL JSON data format includes
+          # the space between the elements.
+          super(serialize(deserialize(raw_old_value)), new_value)
+        end
+      end
+
       class MysqlString < Type::String # :nodoc:
         def serialize(value)
           case value
@@ -1063,6 +1078,8 @@ module ActiveRecord
         end
       end
 
+      ActiveRecord::Type.register(:json, MysqlJson, adapter: :mysql)
+      ActiveRecord::Type.register(:json, MysqlJson, adapter: :mysql2)
       ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql)
       ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql2)
     end
diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb
index b7db57c9fe..ff43c7ec42 100644
--- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb
+++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb
@@ -41,6 +41,10 @@ module ActiveRecord
         true
       end
 
+      def supports_json?
+        version >= '5.7.8'
+      end
+
       # HELPER METHODS ===========================================
 
       def each_hash(result) # :nodoc:
diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb
index 68752cdd80..37226831dc 100644
--- a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb
+++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb
@@ -8,7 +8,6 @@ require 'active_record/connection_adapters/postgresql/oid/decimal'
 require 'active_record/connection_adapters/postgresql/oid/enum'
 require 'active_record/connection_adapters/postgresql/oid/hstore'
 require 'active_record/connection_adapters/postgresql/oid/inet'
-require 'active_record/connection_adapters/postgresql/oid/json'
 require 'active_record/connection_adapters/postgresql/oid/jsonb'
 require 'active_record/connection_adapters/postgresql/oid/money'
 require 'active_record/connection_adapters/postgresql/oid/point'
diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb
deleted file mode 100644
index 8e1256baad..0000000000
--- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-module ActiveRecord
-  module ConnectionAdapters
-    module PostgreSQL
-      module OID # :nodoc:
-        class Json < Type::Value # :nodoc:
-          include Type::Helpers::Mutable
-
-          def type
-            :json
-          end
-
-          def deserialize(value)
-            if value.is_a?(::String)
-              ::ActiveSupport::JSON.decode(value) rescue nil
-            else
-              value
-            end
-          end
-
-          def serialize(value)
-            if value.is_a?(::Array) || value.is_a?(::Hash)
-              ::ActiveSupport::JSON.encode(value)
-            else
-              value
-            end
-          end
-
-          def accessor
-            ActiveRecord::Store::StringKeyedHashAccessor
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb
index 87391b5dc7..1f6d63582c 100644
--- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb
+++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb
@@ -2,7 +2,7 @@ module ActiveRecord
   module ConnectionAdapters
     module PostgreSQL
       module OID # :nodoc:
-        class Jsonb < Json # :nodoc:
+        class Jsonb < Type::Json # :nodoc:
           def type
             :jsonb
           end
diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb
index 2c43c46a3d..861edbf3a2 100644
--- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb
+++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb
@@ -201,6 +201,10 @@ module ActiveRecord
         true
       end
 
+      def supports_json?
+        postgresql_version >= 90200
+      end
+
       def index_algorithms
         { concurrently: 'CONCURRENTLY' }
       end
@@ -478,7 +482,7 @@ module ActiveRecord
           m.register_type 'bytea', OID::Bytea.new
           m.register_type 'point', OID::Point.new
           m.register_type 'hstore', OID::Hstore.new
-          m.register_type 'json', OID::Json.new
+          m.register_type 'json', Type::Json.new
           m.register_type 'jsonb', OID::Jsonb.new
           m.register_type 'cidr', OID::Cidr.new
           m.register_type 'inet', OID::Inet.new
@@ -834,7 +838,6 @@ module ActiveRecord
         ActiveRecord::Type.register(:enum, OID::Enum, adapter: :postgresql)
         ActiveRecord::Type.register(:hstore, OID::Hstore, adapter: :postgresql)
         ActiveRecord::Type.register(:inet, OID::Inet, adapter: :postgresql)
-        ActiveRecord::Type.register(:json, OID::Json, adapter: :postgresql)
         ActiveRecord::Type.register(:jsonb, OID::Jsonb, adapter: :postgresql)
         ActiveRecord::Type.register(:money, OID::Money, adapter: :postgresql)
         ActiveRecord::Type.register(:point, OID::Point, adapter: :postgresql)
diff --git a/activerecord/lib/active_record/type.rb b/activerecord/lib/active_record/type.rb
index 2c0cda69d0..8f56a37f3c 100644
--- a/activerecord/lib/active_record/type.rb
+++ b/activerecord/lib/active_record/type.rb
@@ -10,6 +10,7 @@ require 'active_record/type/decimal'
 require 'active_record/type/decimal_without_scale'
 require 'active_record/type/float'
 require 'active_record/type/integer'
+require 'active_record/type/json'
 require 'active_record/type/serialized'
 require 'active_record/type/string'
 require 'active_record/type/text'
@@ -59,6 +60,7 @@ module ActiveRecord
     register(:decimal, Type::Decimal, override: false)
     register(:float, Type::Float, override: false)
     register(:integer, Type::Integer, override: false)
+    register(:json, Type::Json, override: false)
     register(:string, Type::String, override: false)
     register(:text, Type::Text, override: false)
     register(:time, Type::Time, override: false)
diff --git a/activerecord/lib/active_record/type/json.rb b/activerecord/lib/active_record/type/json.rb
new file mode 100644
index 0000000000..1728bd3a8e
--- /dev/null
+++ b/activerecord/lib/active_record/type/json.rb
@@ -0,0 +1,31 @@
+module ActiveRecord
+  module Type
+    class Json < Type::Value # :nodoc:
+      include Type::Helpers::Mutable
+
+      def type
+        :json
+      end
+
+      def deserialize(value)
+        if value.is_a?(::String)
+          ::ActiveSupport::JSON.decode(value) rescue nil
+        else
+          value
+        end
+      end
+
+      def serialize(value)
+        if value.is_a?(::Array) || value.is_a?(::Hash)
+          ::ActiveSupport::JSON.encode(value)
+        else
+          value
+        end
+      end
+
+      def accessor
+        ActiveRecord::Store::StringKeyedHashAccessor
+      end
+    end
+  end
+end
diff --git a/activerecord/test/cases/adapters/mysql2/json_test.rb b/activerecord/test/cases/adapters/mysql2/json_test.rb
new file mode 100644
index 0000000000..c8c933af5e
--- /dev/null
+++ b/activerecord/test/cases/adapters/mysql2/json_test.rb
@@ -0,0 +1,172 @@
+require 'cases/helper'
+require 'support/schema_dumping_helper'
+
+if ActiveRecord::Base.connection.supports_json?
+class Mysql2JSONTest < ActiveRecord::Mysql2TestCase
+  include SchemaDumpingHelper
+  self.use_transactional_tests = false
+
+  class JsonDataType < ActiveRecord::Base
+    self.table_name = 'json_data_type'
+
+    store_accessor :settings, :resolution
+  end
+
+  def setup
+    @connection = ActiveRecord::Base.connection
+    begin
+      @connection.create_table('json_data_type') do |t|
+        t.json 'payload'
+        t.json 'settings'
+      end
+    end
+  end
+
+  def teardown
+    @connection.drop_table :json_data_type, if_exists: true
+    JsonDataType.reset_column_information
+  end
+
+  def test_column
+    column = JsonDataType.columns_hash["payload"]
+    assert_equal :json, column.type
+    assert_equal 'json', column.sql_type
+
+    type = JsonDataType.type_for_attribute("payload")
+    assert_not type.binary?
+  end
+
+  def test_change_table_supports_json
+    @connection.change_table('json_data_type') do |t|
+      t.json 'users'
+    end
+    JsonDataType.reset_column_information
+    column = JsonDataType.columns_hash['users']
+    assert_equal :json, column.type
+  end
+
+  def test_schema_dumping
+    output = dump_table_schema("json_data_type")
+    assert_match(/t\.json\s+"settings"/, output)
+  end
+
+  def test_cast_value_on_write
+    x = JsonDataType.new payload: {"string" => "foo", :symbol => :bar}
+    assert_equal({"string" => "foo", :symbol => :bar}, x.payload_before_type_cast)
+    assert_equal({"string" => "foo", "symbol" => "bar"}, x.payload)
+    x.save
+    assert_equal({"string" => "foo", "symbol" => "bar"}, x.reload.payload)
+  end
+
+  def test_type_cast_json
+    type = JsonDataType.type_for_attribute("payload")
+
+    data = "{\"a_key\":\"a_value\"}"
+    hash = type.deserialize(data)
+    assert_equal({'a_key' => 'a_value'}, hash)
+    assert_equal({'a_key' => 'a_value'}, type.deserialize(data))
+
+    assert_equal({}, type.deserialize("{}"))
+    assert_equal({'key'=>nil}, type.deserialize('{"key": null}'))
+    assert_equal({'c'=>'}','"a"'=>'b "a b'}, type.deserialize(%q({"c":"}", "\"a\"":"b \"a b"})))
+  end
+
+  def test_rewrite
+    @connection.execute "insert into json_data_type (payload) VALUES ('{\"k\":\"v\"}')"
+    x = JsonDataType.first
+    x.payload = { '"a\'' => 'b' }
+    assert x.save!
+  end
+
+  def test_select
+    @connection.execute "insert into json_data_type (payload) VALUES ('{\"k\":\"v\"}')"
+    x = JsonDataType.first
+    assert_equal({'k' => 'v'}, x.payload)
+  end
+
+  def test_select_multikey
+    @connection.execute %q|insert into json_data_type (payload) VALUES ('{"k1":"v1", "k2":"v2", "k3":[1,2,3]}')|
+    x = JsonDataType.first
+    assert_equal({'k1' => 'v1', 'k2' => 'v2', 'k3' => [1,2,3]}, x.payload)
+  end
+
+  def test_null_json
+    @connection.execute %q|insert into json_data_type (payload) VALUES(null)|
+    x = JsonDataType.first
+    assert_equal(nil, x.payload)
+  end
+
+  def test_select_array_json_value
+    @connection.execute %q|insert into json_data_type (payload) VALUES ('["v0",{"k1":"v1"}]')|
+    x = JsonDataType.first
+    assert_equal(['v0', {'k1' => 'v1'}], x.payload)
+  end
+
+  def test_rewrite_array_json_value
+    @connection.execute %q|insert into json_data_type (payload) VALUES ('["v0",{"k1":"v1"}]')|
+    x = JsonDataType.first
+    x.payload = ['v1', {'k2' => 'v2'}, 'v3']
+    assert x.save!
+  end
+
+  def test_with_store_accessors
+    x = JsonDataType.new(resolution: "320×480")
+    assert_equal "320×480", x.resolution
+
+    x.save!
+    x = JsonDataType.first
+    assert_equal "320×480", x.resolution
+
+    x.resolution = "640×1136"
+    x.save!
+
+    x = JsonDataType.first
+    assert_equal "640×1136", x.resolution
+  end
+
+  def test_duplication_with_store_accessors
+    x = JsonDataType.new(resolution: "320×480")
+    assert_equal "320×480", x.resolution
+
+    y = x.dup
+    assert_equal "320×480", y.resolution
+  end
+
+  def test_yaml_round_trip_with_store_accessors
+    x = JsonDataType.new(resolution: "320×480")
+    assert_equal "320×480", x.resolution
+
+    y = YAML.load(YAML.dump(x))
+    assert_equal "320×480", y.resolution
+  end
+
+  def test_changes_in_place
+    json = JsonDataType.new
+    assert_not json.changed?
+
+    json.payload = { 'one' => 'two' }
+    assert json.changed?
+    assert json.payload_changed?
+
+    json.save!
+    assert_not json.changed?
+
+    json.payload['three'] = 'four'
+    assert json.payload_changed?
+
+    json.save!
+    json.reload
+
+    assert_equal({ 'one' => 'two', 'three' => 'four' }, json.payload)
+    assert_not json.changed?
+  end
+
+  def test_assigning_invalid_json
+    json = JsonDataType.new
+
+    json.payload = 'foo'
+
+    assert_nil json.payload
+  end
+end
+end