Merge branch 'master' into fix-as-timezone-all

13 files changed, 155 insertions, 26 deletions

diff --git a/actionpack/CHANGELOG.md b/actionpack/CHANGELOG.md
index 152ec3700b..baa58c7df2 100644
--- a/actionpack/CHANGELOG.md
+++ b/actionpack/CHANGELOG.md
@@ -1,3 +1,9 @@
+*   Output only one Content-Security-Policy nonce header value per request.
+
+    Fixes #35297.
+
+    *Andrey Novikov*, *Andrew White*
+
 *   Move default headers configuration into their own module that can be included in controllers.
 
     *Kevin Deisz*
diff --git a/actionpack/lib/action_dispatch/http/content_security_policy.rb b/actionpack/lib/action_dispatch/http/content_security_policy.rb
index c1f80a1ffc..17e72b46ff 100644
--- a/actionpack/lib/action_dispatch/http/content_security_policy.rb
+++ b/actionpack/lib/action_dispatch/http/content_security_policy.rb
@@ -21,13 +21,8 @@ module ActionDispatch #:nodoc:
         return response if policy_present?(headers)
 
         if policy = request.content_security_policy
-          if policy.directives["script-src"]
-            if nonce = request.content_security_policy_nonce
-              policy.directives["script-src"] << "'nonce-#{nonce}'"
-            end
-          end
-
-          headers[header_name(request)] = policy.build(request.controller_instance)
+          nonce = request.content_security_policy_nonce
+          headers[header_name(request)] = policy.build(request.controller_instance, nonce)
         end
 
         response
@@ -136,7 +131,9 @@ module ActionDispatch #:nodoc:
       worker_src:      "worker-src"
     }.freeze
 
-    private_constant :MAPPINGS, :DIRECTIVES
+    NONCE_DIRECTIVES = %w[script-src].freeze
+
+    private_constant :MAPPINGS, :DIRECTIVES, :NONCE_DIRECTIVES
 
     attr_reader :directives
 
@@ -205,8 +202,8 @@ module ActionDispatch #:nodoc:
       end
     end
 
-    def build(context = nil)
-      build_directives(context).compact.join("; ")
+    def build(context = nil, nonce = nil)
+      build_directives(context, nonce).compact.join("; ")
     end
 
     private
@@ -229,10 +226,14 @@ module ActionDispatch #:nodoc:
         end
       end
 
-      def build_directives(context)
+      def build_directives(context, nonce)
         @directives.map do |directive, sources|
           if sources.is_a?(Array)
-            "#{directive} #{build_directive(sources, context).join(' ')}"
+            if nonce && nonce_directive?(directive)
+              "#{directive} #{build_directive(sources, context).join(' ')} 'nonce-#{nonce}'"
+            else
+              "#{directive} #{build_directive(sources, context).join(' ')}"
+            end
           elsif sources
             directive
           else
@@ -261,5 +262,9 @@ module ActionDispatch #:nodoc:
           raise RuntimeError, "Unexpected content security policy source: #{source.inspect}"
         end
       end
+
+      def nonce_directive?(directive)
+        NONCE_DIRECTIVES.include?(directive)
+      end
   end
 end
diff --git a/actionpack/test/dispatch/content_security_policy_test.rb b/actionpack/test/dispatch/content_security_policy_test.rb
index 95fce39dad..c4c7f53903 100644
--- a/actionpack/test/dispatch/content_security_policy_test.rb
+++ b/actionpack/test/dispatch/content_security_policy_test.rb
@@ -200,7 +200,7 @@ class ContentSecurityPolicyTest < ActiveSupport::TestCase
   end
 
   def test_dynamic_directives
-    request = Struct.new(:host).new("www.example.com")
+    request = ActionDispatch::Request.new("HTTP_HOST" => "www.example.com")
     controller = Struct.new(:request).new(request)
 
     @policy.script_src -> { request.host }
@@ -209,7 +209,9 @@ class ContentSecurityPolicyTest < ActiveSupport::TestCase
 
   def test_mixed_static_and_dynamic_directives
     @policy.script_src :self, -> { "foo.com" }, "bar.com"
-    assert_equal "script-src 'self' foo.com bar.com", @policy.build(Object.new)
+    request = ActionDispatch::Request.new({})
+    controller = Struct.new(:request).new(request)
+    assert_equal "script-src 'self' foo.com bar.com", @policy.build(controller)
   end
 
   def test_invalid_directive_source
@@ -241,6 +243,73 @@ class ContentSecurityPolicyTest < ActiveSupport::TestCase
   end
 end
 
+class DefaultContentSecurityPolicyIntegrationTest < ActionDispatch::IntegrationTest
+  class PolicyController < ActionController::Base
+    def index
+      head :ok
+    end
+  end
+
+  ROUTES = ActionDispatch::Routing::RouteSet.new
+  ROUTES.draw do
+    scope module: "default_content_security_policy_integration_test" do
+      get "/", to: "policy#index"
+    end
+  end
+
+  POLICY = ActionDispatch::ContentSecurityPolicy.new do |p|
+    p.default_src :self
+    p.script_src  :https
+  end
+
+  class PolicyConfigMiddleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      env["action_dispatch.content_security_policy"] = POLICY
+      env["action_dispatch.content_security_policy_nonce_generator"] = proc { "iyhD0Yc0W+c=" }
+      env["action_dispatch.content_security_policy_report_only"] = false
+      env["action_dispatch.show_exceptions"] = false
+
+      @app.call(env)
+    end
+  end
+
+  APP = build_app(ROUTES) do |middleware|
+    middleware.use PolicyConfigMiddleware
+    middleware.use ActionDispatch::ContentSecurityPolicy::Middleware
+  end
+
+  def app
+    APP
+  end
+
+  def test_adds_nonce_to_script_src_content_security_policy_only_once
+    get "/"
+    get "/"
+    assert_policy "default-src 'self'; script-src https: 'nonce-iyhD0Yc0W+c='"
+  end
+
+  private
+
+    def assert_policy(expected, report_only: false)
+      assert_response :success
+
+      if report_only
+        expected_header = "Content-Security-Policy-Report-Only"
+        unexpected_header = "Content-Security-Policy"
+      else
+        expected_header = "Content-Security-Policy"
+        unexpected_header = "Content-Security-Policy-Report-Only"
+      end
+
+      assert_nil response.headers[unexpected_header]
+      assert_equal expected, response.headers[expected_header]
+    end
+end
+
 class ContentSecurityPolicyIntegrationTest < ActionDispatch::IntegrationTest
   class PolicyController < ActionController::Base
     content_security_policy only: :inline do |p|
diff --git a/actionview/CHANGELOG.md b/actionview/CHANGELOG.md
index 6bb1efb0ac..d833f9cd98 100644
--- a/actionview/CHANGELOG.md
+++ b/actionview/CHANGELOG.md
@@ -1,3 +1,9 @@
+*   Add the `nonce: true` option for `javascript_include_tag` helper to
+    support automatic nonce generation for Content Security Policy.
+    Works the same way as `javascript_tag nonce: true` does.
+
+    *Yaroslav Markin*
+
 *   Remove `ActionView::Helpers::RecordTagHelper`.
 
     *Yoshiyuki Hirano*
diff --git a/actionview/lib/action_view/digestor.rb b/actionview/lib/action_view/digestor.rb
index 45cf48b3e0..3832293251 100644
--- a/actionview/lib/action_view/digestor.rb
+++ b/actionview/lib/action_view/digestor.rb
@@ -71,11 +71,16 @@ module ActionView
 
       private
         def find_template(finder, *args)
+          name = args.first
+          prefixes = args[1] || []
+          partial = args[2] || false
+          keys = args[3] || []
+          options = args[4] || {}
           finder.disable_cache do
             if format = finder.rendered_format
-              finder.find_all(*args, formats: [format]).first || finder.find_all(*args).first
+              finder.find_all(name, prefixes, partial, keys, options.merge(formats: [format])).first || finder.find_all(name, prefixes, partial, keys, options).first
             else
-              finder.find_all(*args).first
+              finder.find_all(name, prefixes, partial, keys, options).first
             end
           end
         end
diff --git a/actionview/lib/action_view/helpers/asset_tag_helper.rb b/actionview/lib/action_view/helpers/asset_tag_helper.rb
index 06fa1875fc..257080d902 100644
--- a/actionview/lib/action_view/helpers/asset_tag_helper.rb
+++ b/actionview/lib/action_view/helpers/asset_tag_helper.rb
@@ -55,6 +55,8 @@ module ActionView
       #   that path.
       # * <tt>:skip_pipeline</tt>  - This option is used to bypass the asset pipeline
       #   when it is set to true.
+      # * <tt>:nonce<tt>  - When set to true, adds an automatic nonce value if
+      #   you have Content Security Policy enabled.
       #
       # ==== Examples
       #
@@ -79,6 +81,9 @@ module ActionView
       #
       #   javascript_include_tag "http://www.example.com/xmlhr.js"
       #   # => <script src="http://www.example.com/xmlhr.js"></script>
+      #
+      #   javascript_include_tag "http://www.example.com/xmlhr.js", nonce: true
+      #   # => <script src="http://www.example.com/xmlhr.js" nonce="..."></script>
       def javascript_include_tag(*sources)
         options = sources.extract_options!.stringify_keys
         path_options = options.extract!("protocol", "extname", "host", "skip_pipeline").symbolize_keys
@@ -90,6 +95,9 @@ module ActionView
           tag_options = {
             "src" => href
           }.merge!(options)
+          if tag_options["nonce"] == true
+            tag_options["nonce"] = content_security_policy_nonce
+          end
           content_tag("script".freeze, "", tag_options)
         }.join("\n").html_safe
 
diff --git a/actionview/lib/action_view/helpers/translation_helper.rb b/actionview/lib/action_view/helpers/translation_helper.rb
index 80cb73d683..db44fdbfee 100644
--- a/actionview/lib/action_view/helpers/translation_helper.rb
+++ b/actionview/lib/action_view/helpers/translation_helper.rb
@@ -60,7 +60,11 @@ module ActionView
       def translate(key, options = {})
         options = options.dup
         has_default = options.has_key?(:default)
-        remaining_defaults = Array(options.delete(:default)).compact
+        if has_default
+          remaining_defaults = Array(options.delete(:default)).compact
+        else
+          remaining_defaults = []
+        end
 
         if has_default && !remaining_defaults.first.kind_of?(Symbol)
           options[:default] = remaining_defaults
diff --git a/actionview/test/template/asset_tag_helper_test.rb b/actionview/test/template/asset_tag_helper_test.rb
index 6d98eacfb8..e68f03d1f4 100644
--- a/actionview/test/template/asset_tag_helper_test.rb
+++ b/actionview/test/template/asset_tag_helper_test.rb
@@ -29,6 +29,10 @@ class AssetTagHelperTest < ActionView::TestCase
     "http://www.example.com"
   end
 
+  def content_security_policy_nonce
+    "iyhD0Yc0W+c="
+  end
+
   AssetPathToTag = {
     %(asset_path(""))             => %(),
     %(asset_path("   "))          => %(),
@@ -421,6 +425,10 @@ class AssetTagHelperTest < ActionView::TestCase
     assert_dom_equal %(<script src="//assets.example.com/javascripts/prototype.js"></script>), javascript_include_tag("prototype")
   end
 
+  def test_javascript_include_tag_nonce
+    assert_dom_equal %(<script src="/javascripts/bank.js" nonce="iyhD0Yc0W+c="></script>), javascript_include_tag("bank", nonce: true)
+  end
+
   def test_stylesheet_path
     StylePathToTag.each { |method, tag| assert_dom_equal(tag, eval(method)) }
   end
diff --git a/activemodel/lib/active_model/validations/inclusion.rb b/activemodel/lib/active_model/validations/inclusion.rb
index 3104e7e329..9c12dc14c5 100644
--- a/activemodel/lib/active_model/validations/inclusion.rb
+++ b/activemodel/lib/active_model/validations/inclusion.rb
@@ -19,7 +19,7 @@ module ActiveModel
       # particular enumerable object.
       #
       #   class Person < ActiveRecord::Base
-      #     validates_inclusion_of :gender, in: %w( m f )
+      #     validates_inclusion_of :role, in: %w( admin contributor )
       #     validates_inclusion_of :age, in: 0..99
       #     validates_inclusion_of :format, in: %w( jpg gif png ), message: "extension %{value} is not included in the list"
       #     validates_inclusion_of :states, in: ->(person) { STATES[person.country] }
diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md
index aa7e206bb2..247eec68ac 100644
--- a/activesupport/CHANGELOG.md
+++ b/activesupport/CHANGELOG.md
@@ -3,6 +3,11 @@
 
     *Dominik Sander*
 
+*   Redis cache store: `delete_matched` no longer blocks the Redis server.
+    (Switches from evaled Lua to a batched SCAN + DEL loop.)
+
+    *Gleb Mazovetskiy*
+
 *   Fix bug where `ActiveSupport::Cache` will massively inflate the storage
     size when compression is enabled (which is true by default). This patch
     does not attempt to repair existing data: please manually flush the cache
diff --git a/activesupport/lib/active_support/cache/redis_cache_store.rb b/activesupport/lib/active_support/cache/redis_cache_store.rb
index 74f935e02e..11c574258f 100644
--- a/activesupport/lib/active_support/cache/redis_cache_store.rb
+++ b/activesupport/lib/active_support/cache/redis_cache_store.rb
@@ -62,8 +62,9 @@ module ActiveSupport
         end
       end
 
-      DELETE_GLOB_LUA = "for i, name in ipairs(redis.call('KEYS', ARGV[1])) do redis.call('DEL', name); end"
-      private_constant :DELETE_GLOB_LUA
+      # The maximum number of entries to receive per SCAN call.
+      SCAN_BATCH_SIZE = 1000
+      private_constant :SCAN_BATCH_SIZE
 
       # Support raw values in the local cache strategy.
       module LocalCacheWithRaw # :nodoc:
@@ -231,12 +232,18 @@ module ActiveSupport
       # Failsafe: Raises errors.
       def delete_matched(matcher, options = nil)
         instrument :delete_matched, matcher do
-          case matcher
-          when String
-            redis.with { |c| c.eval DELETE_GLOB_LUA, [], [namespace_key(matcher, options)] }
-          else
+          unless String === matcher
             raise ArgumentError, "Only Redis glob strings are supported: #{matcher.inspect}"
           end
+          redis.with do |c|
+            pattern = namespace_key(matcher, options)
+            cursor = "0"
+            # Fetch keys in batches using SCAN to avoid blocking the Redis server.
+            begin
+              cursor, keys = c.scan(cursor, match: pattern, count: SCAN_BATCH_SIZE)
+              c.del(*keys) unless keys.empty?
+            end until cursor == "0"
+          end
         end
       end
 
diff --git a/guides/source/asset_pipeline.md b/guides/source/asset_pipeline.md
index 2f5854fed0..88b87b78d2 100644
--- a/guides/source/asset_pipeline.md
+++ b/guides/source/asset_pipeline.md
@@ -728,8 +728,8 @@ Rails.application.config.assets.precompile += %w( admin.js admin.css )
 NOTE. Always specify an expected compiled filename that ends with `.js` or `.css`,
 even if you want to add Sass or CoffeeScript files to the precompile array.
 
-The task also generates a `.sprockets-manifest-md5hash.json` (where `md5hash` is
-an MD5 hash) that contains a list with all your assets and their respective
+The task also generates a `.sprockets-manifest-randomhex.json` (where `randomhex` is
+a 16-byte random hex string) that contains a list with all your assets and their respective
 fingerprints. This is used by the Rails helper methods to avoid handing the
 mapping requests back to Sprockets. A typical manifest file looks like:
 
diff --git a/guides/source/security.md b/guides/source/security.md
index a21526d895..3ac50fb147 100644
--- a/guides/source/security.md
+++ b/guides/source/security.md
@@ -1182,6 +1182,12 @@ as part of `html_options`. Example:
 <% end -%>
 ```
 
+The same works with `javascript_include_tag`:
+
+```html+erb
+<%= javascript_include_tag "script", nonce: true %>
+```
+
 Use [`csp_meta_tag`](http://api.rubyonrails.org/classes/ActionView/Helpers/CspHelper.html#method-i-csp_meta_tag)
 helper to create a meta tag "csp-nonce" with the per-session nonce value
 for allowing inline `<script>` tags.