Merge branch 'master' into laurocaetano-fix_send_file

* master: (536 commits)
  doc, API example on how to use `Model#exists?` with multiple IDs. [ci skip]
  Restore DATABASE_URL even if it's nil in connection_handler test
  [ci skip] - error_messages_for has been deprecated since 2.3.8 - lets reduce any confusion for users
  Ensure Active Record connection consistency
  Revert "ask the fixture set for the sql statements"
  Check `respond_to` before delegation due to: https://github.com/ruby/ruby/commit/d781caaf313b8649948c107bba277e5ad7307314
  Adding Hash#compact and Hash#compact! methods
  MySQL version 4.1 was EOL on December 31, 2009 We should at least recommend modern versions of MySQL to users.
  clear cache on body close so that cache remains during rendering
  add a more restricted codepath for templates fixes #13390
  refactor generator tests to use block form of Tempfile
  Fix typo [ci skip]
  Move finish_template as the last public method in the generator
  Minor typos fix [ci skip]
  make `change_column_null` reversible. Closes #13576.
  create/drop test and development databases only if RAILS_ENV is nil
  Revert "Speedup String#to"
  typo fix in test name. [ci skip].
  `core_ext/string/access.rb` test what we are documenting.
  Fix typo in image_tag documentation
  ...

Conflicts:
	actionpack/CHANGELOG.md

9 files changed, 212 insertions, 41 deletions

diff --git a/actionpack/lib/action_controller/metal/head.rb b/actionpack/lib/action_controller/metal/head.rb
index 424473801d..43407f5b78 100644
--- a/actionpack/lib/action_controller/metal/head.rb
+++ b/actionpack/lib/action_controller/metal/head.rb
@@ -1,6 +1,6 @@
 module ActionController
   module Head
-    # Return a response that has no content (merely headers). The options
+    # Returns a response that has no content (merely headers). The options
     # argument is interpreted to be a hash of header names and values.
     # This allows you to easily return a response that consists only of
     # significant headers:
diff --git a/actionpack/lib/action_controller/metal/instrumentation.rb b/actionpack/lib/action_controller/metal/instrumentation.rb
index d3aa8f90c5..b0e164bc57 100644
--- a/actionpack/lib/action_controller/metal/instrumentation.rb
+++ b/actionpack/lib/action_controller/metal/instrumentation.rb
@@ -67,7 +67,7 @@ module ActionController
 
   private
 
-    # A hook invoked everytime a before callback is halted.
+    # A hook invoked every time a before callback is halted.
     def halted_callback_hook(filter)
       ActiveSupport::Notifications.instrument("halted_callback.action_controller", :filter => filter)
     end
diff --git a/actionpack/lib/action_controller/metal/live.rb b/actionpack/lib/action_controller/metal/live.rb
index 09ea4919c2..33014b97ca 100644
--- a/actionpack/lib/action_controller/metal/live.rb
+++ b/actionpack/lib/action_controller/metal/live.rb
@@ -48,7 +48,7 @@ module ActionController
     #   the server will receive a +Last-Event-ID+ header with value equal to +id+.
     #
     # After setting an option in the constructor of the SSE object, all future
-    # SSEs sent accross the stream will use those options unless overridden.
+    # SSEs sent across the stream will use those options unless overridden.
     #
     # Example Usage:
     #
diff --git a/actionpack/lib/action_controller/metal/mime_responds.rb b/actionpack/lib/action_controller/metal/mime_responds.rb
index 84ade41036..d5e08b7034 100644
--- a/actionpack/lib/action_controller/metal/mime_responds.rb
+++ b/actionpack/lib/action_controller/metal/mime_responds.rb
@@ -181,6 +181,61 @@ module ActionController #:nodoc:
     #     end
     #   end
     #
+    # Formats can have different variants.
+    #
+    # The request variant is a specialization of the request format, like <tt>:tablet</tt>,
+    # <tt>:phone</tt>, or <tt>:desktop</tt>.
+    #
+    # We often want to render different html/json/xml templates for phones,
+    # tablets, and desktop browsers. Variants make it easy.
+    #
+    # You can set the variant in a +before_action+:
+    #
+    #   request.variant = :tablet if request.user_agent =~ /iPad/
+    #
+    # Respond to variants in the action just like you respond to formats:
+    #
+    #   respond_to do |format|
+    #     format.html do |variant|
+    #       variant.tablet # renders app/views/projects/show.html+tablet.erb
+    #       variant.phone { extra_setup; render ... }
+    #       variant.none  { special_setup } # executed only if there is no variant set
+    #     end
+    #   end
+    #
+    # Provide separate templates for each format and variant:
+    #
+    #   app/views/projects/show.html.erb
+    #   app/views/projects/show.html+tablet.erb
+    #   app/views/projects/show.html+phone.erb
+    #
+    # When you're not sharing any code within the format, you can simplify defining variants
+    # using the inline syntax:
+    #
+    #   respond_to do |format|
+    #     format.js         { render "trash" }
+    #     format.html.phone { redirect_to progress_path }
+    #     format.html.none  { render "trash" }
+    #   end
+    # 
+    # Variants also support common `any`/`all` block that formats have.
+    #
+    # It works for both inline:
+    #
+    #   respond_to do |format|
+    #     format.html.any   { render text: "any"   }
+    #     format.html.phone { render text: "phone" }
+    #   end
+    #
+    # and block syntax:
+    #
+    #   respond_to do |format|
+    #     format.html do |variant|
+    #       variant.any(:tablet, :phablet){ render text: "any" }
+    #       variant.phone { render text: "phone" }
+    #     end
+    #   end
+    #
     # Be sure to check the documentation of +respond_with+ and
     # <tt>ActionController::MimeResponds.respond_to</tt> for more examples.
     def respond_to(*mimes, &block)
@@ -260,7 +315,7 @@ module ActionController #:nodoc:
     # * for other requests - i.e. data formats such as xml, json, csv etc, if
     #   the resource passed to +respond_with+ responds to <code>to_<format></code>,
     #   the method attempts to render the resource in the requested format
-    #   directly, e.g. for an xml request, the response is equivalent to calling 
+    #   directly, e.g. for an xml request, the response is equivalent to calling
     #   <code>render xml: resource</code>.
     #
     # === Nested resources
@@ -321,8 +376,10 @@ module ActionController #:nodoc:
     # 2. <tt>:action</tt> - overwrites the default render action used after an
     #    unsuccessful html +post+ request.
     def respond_with(*resources, &block)
-      raise "In order to use respond_with, first you need to declare the formats your " \
-            "controller responds to in the class level" if self.class.mimes_for_respond_to.empty?
+      if self.class.mimes_for_respond_to.empty?
+        raise "In order to use respond_with, first you need to declare the " \
+          "formats your controller responds to in the class level."
+      end
 
       if collector = retrieve_collector_from_mimes(&block)
         options = resources.size == 1 ? {} : resources.extract_options!
@@ -360,7 +417,7 @@ module ActionController #:nodoc:
     # is available.
     def retrieve_collector_from_mimes(mimes=nil, &block) #:nodoc:
       mimes ||= collect_mimes_from_class_level
-      collector = Collector.new(mimes)
+      collector = Collector.new(mimes, request.variant)
       block.call(collector) if block_given?
       format = collector.negotiate_format(request)
 
@@ -398,9 +455,11 @@ module ActionController #:nodoc:
       include AbstractController::Collector
       attr_accessor :format
 
-      def initialize(mimes)
+      def initialize(mimes, variant = nil)
         @responses = {}
-        mimes.each { |mime| send(mime) }
+        @variant = variant
+
+        mimes.each { |mime| @responses["Mime::#{mime.upcase}".constantize] = nil }
       end
 
       def any(*args, &block)
@@ -414,16 +473,63 @@ module ActionController #:nodoc:
 
       def custom(mime_type, &block)
         mime_type = Mime::Type.lookup(mime_type.to_s) unless mime_type.is_a?(Mime::Type)
-        @responses[mime_type] ||= block
+        @responses[mime_type] ||= if block_given?
+          block
+        else
+          VariantCollector.new(@variant)
+        end
       end
 
       def response
-        @responses.fetch(format, @responses[Mime::ALL])
+        response = @responses.fetch(format, @responses[Mime::ALL])
+        if response.is_a?(VariantCollector) # `format.html.phone` - variant inline syntax
+          response.variant
+        elsif response.nil? || response.arity == 0 # `format.html` - just a format, call its block
+          response
+        else # `format.html{ |variant| variant.phone }` - variant block syntax
+          variant_collector = VariantCollector.new(@variant)
+          response.call(variant_collector) #call format block with variants collector
+          variant_collector.variant
+        end
       end
 
       def negotiate_format(request)
         @format = request.negotiate_mime(@responses.keys)
       end
+
+      class VariantCollector #:nodoc:
+        def initialize(variant = nil)
+          @variant = variant
+          @variants = {}
+        end
+
+        def any(*args, &block)
+          if block_given?
+            if args.any? && args.none?{ |a| a == @variant }
+              args.each{ |v| @variants[v] = block }
+            else
+              @variants[:any] = block
+            end
+          end
+        end
+        alias :all :any
+
+        def method_missing(name, *args, &block)
+          @variants[name] = block if block_given?
+        end
+
+        def variant
+          key = if @variant.nil?
+            :none
+          elsif @variants.has_key?(@variant)
+            @variant
+          else
+            :any
+          end
+
+          @variants[key]
+        end
+      end
     end
   end
 end
diff --git a/actionpack/lib/action_controller/metal/redirecting.rb b/actionpack/lib/action_controller/metal/redirecting.rb
index ab14a61b97..2812038938 100644
--- a/actionpack/lib/action_controller/metal/redirecting.rb
+++ b/actionpack/lib/action_controller/metal/redirecting.rb
@@ -58,7 +58,7 @@ module ActionController
     #   redirect_to post_url(@post), alert: "Watch it, mister!"
     #   redirect_to post_url(@post), status: :found, notice: "Pay attention to the road"
     #   redirect_to post_url(@post), status: 301, flash: { updated_post_id: @post.id }
-    #   redirect_to { action: 'atom' }, alert: "Something serious happened"
+    #   redirect_to({ action: 'atom' }, alert: "Something serious happened")
     #
     # When using <tt>redirect_to :back</tt>, if there is no referrer, ActionController::RedirectBackError will be raised. You may specify some fallback
     # behavior for this case by rescuing ActionController::RedirectBackError.
diff --git a/actionpack/lib/action_controller/metal/renderers.rb b/actionpack/lib/action_controller/metal/renderers.rb
index 62a3844b04..6c7b4652d4 100644
--- a/actionpack/lib/action_controller/metal/renderers.rb
+++ b/actionpack/lib/action_controller/metal/renderers.rb
@@ -43,7 +43,7 @@ module ActionController
     end
 
     # Hash of available renderers, mapping a renderer name to its proc.
-    # Default keys are :json, :js, :xml.
+    # Default keys are <tt>:json</tt>, <tt>:js</tt>, <tt>:xml</tt>.
     RENDERERS = Set.new
 
     # Adds a new renderer to call within controller actions.
diff --git a/actionpack/lib/action_controller/metal/request_forgery_protection.rb b/actionpack/lib/action_controller/metal/request_forgery_protection.rb
index bd64b1f812..c88074d4c6 100644
--- a/actionpack/lib/action_controller/metal/request_forgery_protection.rb
+++ b/actionpack/lib/action_controller/metal/request_forgery_protection.rb
@@ -5,14 +5,24 @@ module ActionController #:nodoc:
   class InvalidAuthenticityToken < ActionControllerError #:nodoc:
   end
 
+  class InvalidCrossOriginRequest < ActionControllerError #:nodoc:
+  end
+
   # Controller actions are protected from Cross-Site Request Forgery (CSRF) attacks
   # by including a token in the rendered html for your application. This token is
   # stored as a random string in the session, to which an attacker does not have
   # access. When a request reaches your application, \Rails verifies the received
   # token with the token in the session. Only HTML and JavaScript requests are checked,
   # so this will not protect your XML API (presumably you'll have a different
-  # authentication scheme there anyway). Also, GET requests are not protected as these
-  # should be idempotent.
+  # authentication scheme there anyway).
+  #
+  # GET requests are not protected since they don't have side effects like writing
+  # to the database and don't leak sensitive information. JavaScript requests are
+  # an exception: a third-party site can use a <script> tag to reference a JavaScript
+  # URL on your site. When your JavaScript response loads on their site, it executes.
+  # With carefully crafted JavaScript on their end, sensitive data in your JavaScript
+  # response may be extracted. To prevent this, only XmlHttpRequest (known as XHR or
+  # Ajax) requests are allowed to make GET requests for JavaScript responses.
   #
   # It's important to remember that XML or JSON requests are also affected and if
   # you're building an API you'll need something like:
@@ -65,17 +75,16 @@ module ActionController #:nodoc:
     module ClassMethods
       # Turn on request forgery protection. Bear in mind that only non-GET, HTML/JavaScript requests are checked.
       #
+      #   class ApplicationController < ActionController::Base
+      #     protect_from_forgery
+      #   end
+      #
       #   class FooController < ApplicationController
       #     protect_from_forgery except: :index
       #
-      # You can disable csrf protection on controller-by-controller basis:
-      #
+      # You can disable CSRF protection on controller by skipping the verification before_action:
       #   skip_before_action :verify_authenticity_token
       #
-      # It can also be disabled for specific controller actions:
-      #
-      #   skip_before_action :verify_authenticity_token, except: [:create]
-      #
       # Valid Options:
       #
       # * <tt>:only/:except</tt> - Passed to the <tt>before_action</tt> call. Set which actions are verified.
@@ -89,6 +98,7 @@ module ActionController #:nodoc:
         self.forgery_protection_strategy = protection_method_class(options[:with] || :null_session)
         self.request_forgery_protection_token ||= :authenticity_token
         prepend_before_action :verify_authenticity_token, options
+        append_after_action :verify_same_origin_request
       end
 
       private
@@ -169,18 +179,61 @@ module ActionController #:nodoc:
     end
 
     protected
+      # The actual before_action that is used to verify the CSRF token.
+      # Don't override this directly. Provide your own forgery protection
+      # strategy instead. If you override, you'll disable same-origin
+      # `<script>` verification.
+      #
+      # Lean on the protect_from_forgery declaration to mark which actions are
+      # due for same-origin request verification. If protect_from_forgery is
+      # enabled on an action, this before_action flags its after_action to
+      # verify that JavaScript responses are for XHR requests, ensuring they
+      # follow the browser's same-origin policy.
+      def verify_authenticity_token
+        mark_for_same_origin_verification!
+
+        if !verified_request?
+          logger.warn "Can't verify CSRF token authenticity" if logger
+          handle_unverified_request
+        end
+      end
+
       def handle_unverified_request
         forgery_protection_strategy.new(self).handle_unverified_request
       end
 
-      # The actual before_action that is used. Modify this to change how you handle unverified requests.
-      def verify_authenticity_token
-        unless verified_request?
-          logger.warn "Can't verify CSRF token authenticity" if logger
-          handle_unverified_request
+      CROSS_ORIGIN_JAVASCRIPT_WARNING = "Security warning: an embedded " \
+        "<script> tag on another site requested protected JavaScript. " \
+        "If you know what you're doing, go ahead and disable forgery " \
+        "protection on this action to permit cross-origin JavaScript embedding."
+      private_constant :CROSS_ORIGIN_JAVASCRIPT_WARNING
+
+      # If `verify_authenticity_token` was run (indicating that we have
+      # forgery protection enabled for this request) then also verify that
+      # we aren't serving an unauthorized cross-origin response.
+      def verify_same_origin_request
+        if marked_for_same_origin_verification? && non_xhr_javascript_response?
+          logger.warn CROSS_ORIGIN_JAVASCRIPT_WARNING if logger
+          raise ActionController::InvalidCrossOriginRequest, CROSS_ORIGIN_JAVASCRIPT_WARNING
         end
       end
 
+      # GET requests are checked for cross-origin JavaScript after rendering.
+      def mark_for_same_origin_verification!
+        @marked_for_same_origin_verification = request.get?
+      end
+
+      # If the `verify_authenticity_token` before_action ran, verify that
+      # JavaScript responses are only served to same-origin GET requests.
+      def marked_for_same_origin_verification?
+        @marked_for_same_origin_verification ||= false
+      end
+
+      # Check for cross-origin JavaScript responses.
+      def non_xhr_javascript_response?
+        content_type =~ %r(\Atext/javascript) && !request.xhr?
+      end
+
       # Returns true or false if a request is verified. Checks:
       #
       # * is it a GET or HEAD request?  Gets should be safe and idempotent
diff --git a/actionpack/lib/action_controller/metal/responder.rb b/actionpack/lib/action_controller/metal/responder.rb
index b4ba169e8f..e24b56fa91 100644
--- a/actionpack/lib/action_controller/metal/responder.rb
+++ b/actionpack/lib/action_controller/metal/responder.rb
@@ -270,7 +270,7 @@ module ActionController #:nodoc:
       resource.respond_to?(:errors) && !resource.errors.empty?
     end
 
-    # Check whether the neceessary Renderer is available
+    # Check whether the necessary Renderer is available
     def has_renderer?
       Renderers::RENDERERS.include?(format)
     end
diff --git a/actionpack/lib/action_controller/metal/strong_parameters.rb b/actionpack/lib/action_controller/metal/strong_parameters.rb
index b4948d99a8..48a916f2b1 100644
--- a/actionpack/lib/action_controller/metal/strong_parameters.rb
+++ b/actionpack/lib/action_controller/metal/strong_parameters.rb
@@ -3,6 +3,7 @@ require 'active_support/core_ext/array/wrap'
 require 'active_support/rescuable'
 require 'action_dispatch/http/upload'
 require 'stringio'
+require 'set'
 
 module ActionController
   # Raised when a required parameter is missing.
@@ -125,6 +126,13 @@ module ActionController
       @permitted = self.class.permit_all_parameters
     end
 
+    # Attribute that keeps track of converted arrays, if any, to avoid double
+    # looping in the common use case permit + mass-assignment. Defined in a
+    # method to instantiate it only if needed.
+    def converted_arrays
+      @converted_arrays ||= Set.new
+    end
+
     # Returns +true+ if the parameter is permitted, +false+ otherwise.
     #
     #   params = ActionController::Parameters.new
@@ -149,8 +157,10 @@ module ActionController
     #   Person.new(params) # => #<Person id: nil, name: "Francesco">
     def permit!
       each_pair do |key, value|
-        convert_hashes_to_parameters(key, value)
-        self[key].permit! if self[key].respond_to? :permit!
+        value = convert_hashes_to_parameters(key, value)
+        Array.wrap(value).each do |_|
+          _.permit! if _.respond_to? :permit!
+        end
       end
 
       @permitted = true
@@ -284,14 +294,7 @@ module ActionController
     #   params.fetch(:none, 'Francesco')    # => "Francesco"
     #   params.fetch(:none) { 'Francesco' } # => "Francesco"
     def fetch(key, *args)
-      value = super
-      # Don't rely on +convert_hashes_to_parameters+
-      # so as to not mutate via a +fetch+
-      if value.is_a?(Hash)
-        value = self.class.new(value)
-        value.permit! if permitted?
-      end
-      value
+      convert_hashes_to_parameters(key, super, false)
     rescue KeyError
       raise ActionController::ParameterMissing.new(key)
     end
@@ -329,12 +332,21 @@ module ActionController
       end
 
     private
-      def convert_hashes_to_parameters(key, value)
-        if value.is_a?(Parameters) || !value.is_a?(Hash)
+      def convert_hashes_to_parameters(key, value, assign_if_converted=true)
+        converted = convert_value_to_parameters(value)
+        self[key] = converted if assign_if_converted && !converted.equal?(value)
+        converted
+      end
+
+      def convert_value_to_parameters(value)
+        if value.is_a?(Array) && !converted_arrays.member?(value)
+          converted = value.map { |_| convert_value_to_parameters(_) }
+          converted_arrays << converted
+          converted
+        elsif value.is_a?(Parameters) || !value.is_a?(Hash)
           value
         else
-          # Convert to Parameters on first access
-          self[key] = self.class.new(value)
+          self.class.new(value)
         end
       end