Merge pull request #32125 from rails/switch-to-not-enforcing-utf8-by-default

Don't enforce UTF-8 by default

actionview/CHANGELOG.md

 ## Rails 6.0.0.alpha (Unreleased) ##
 
+*   Don't enforce UTF-8 by default
+
+    With the disabling of TLS 1.0 by most major websites, continuing to run
+    IE8 or lower becomes increasingly difficult so default to not enforcing
+    UTF-8 encoding as it's not relevant to other browsers.
+
+    *Andrew White*
+
 *   Change translation key of `submit_tag` from `module_name_class_name` to `module_name/class_name`.
 
     *Rui Onodera*

actionview/lib/action_view/helpers/form_helper.rb

@@ -1519,10 +1519,10 @@ def range_field(object_name, method, options = {})
 
       private
         def html_options_for_form_with(url_for_options = nil, model = nil, html: {}, local: !form_with_generates_remote_forms,
-          skip_enforcing_utf8: false, **options)
+          skip_enforcing_utf8: nil, **options)
           html_options = options.slice(:id, :class, :multipart, :method, :data).merge(html)
           html_options[:method] ||= :patch if model.respond_to?(:persisted?) && model.persisted?
-          html_options[:enforce_utf8] = !skip_enforcing_utf8
+          html_options[:enforce_utf8] = !skip_enforcing_utf8 unless skip_enforcing_utf8.nil?
 
           html_options[:enctype] = "multipart/form-data" if html_options.delete(:multipart)
 

actionview/lib/action_view/helpers/form_tag_helper.rb

@@ -22,6 +22,8 @@ module FormTagHelper
       mattr_accessor :embed_authenticity_token_in_remote_forms
       self.embed_authenticity_token_in_remote_forms = nil
 
+      mattr_accessor :default_enforce_utf8, default: true
+
       # Starts a form tag that points the action to a url configured with <tt>url_for_options</tt> just like
       # ActionController::Base#url_for. The method for the form defaults to POST.
       #
@@ -866,7 +868,7 @@ def extra_tags_for_form(html_options)
               })
             end
 
-          if html_options.delete("enforce_utf8") { true }
+          if html_options.delete("enforce_utf8") { default_enforce_utf8 }
             utf8_enforcer_tag + method_tag
           else
             method_tag

actionview/lib/action_view/railtie.rb

@@ -9,6 +9,7 @@ class Railtie < Rails::Engine # :nodoc:
     config.action_view = ActiveSupport::OrderedOptions.new
     config.action_view.embed_authenticity_token_in_remote_forms = nil
     config.action_view.debug_missing_translation = true
+    config.action_view.default_enforce_utf8 = nil
 
     config.eager_load_namespaces << ActionView
 
@@ -35,6 +36,15 @@ class Railtie < Rails::Engine # :nodoc:
       end
     end
 
+    initializer "action_view.default_enforce_utf8" do |app|
+      ActiveSupport.on_load(:action_view) do
+        default_enforce_utf8 = app.config.action_view.delete(:default_enforce_utf8)
+        unless default_enforce_utf8.nil?
+          ActionView::Helpers::FormTagHelper.default_enforce_utf8 = default_enforce_utf8
+        end
+      end
+    end
+
     initializer "action_view.logger" do
       ActiveSupport.on_load(:action_view) { self.logger ||= Rails.logger }
     end

actionview/test/template/form_helper/form_with_test.rb

@@ -14,6 +14,16 @@ class FormWithTest < ActionView::TestCase
   teardown do
     ActionView::Helpers::FormHelper.form_with_generates_ids = @old_value
   end
+
+  private
+    def with_default_enforce_utf8(value)
+      old_value = ActionView::Helpers::FormTagHelper.default_enforce_utf8
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = value
+
+      yield
+    ensure
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = old_value
+    end
 end
 
 class FormWithActsLikeFormTagTest < FormWithTest
@@ -111,6 +121,24 @@ def test_form_with_skip_enforcing_utf8_true
     assert_predicate actual, :html_safe?
   end
 
+  def test_form_with_default_enforce_utf8_false
+    with_default_enforce_utf8 false do
+      actual = form_with
+      expected = whole_form("http://www.example.com", skip_enforcing_utf8: true)
+      assert_dom_equal expected, actual
+      assert_predicate actual, :html_safe?
+    end
+  end
+
+  def test_form_with_default_enforce_utf8_true
+    with_default_enforce_utf8 true do
+      actual = form_with
+      expected = whole_form("http://www.example.com", skip_enforcing_utf8: false)
+      assert_dom_equal expected, actual
+      assert_predicate actual, :html_safe?
+    end
+  end
+
   def test_form_with_with_block_in_erb
     output_buffer = render_erb("<%= form_with(url: 'http://www.example.com') do %>Hello world!<% end %>")
 
@@ -819,6 +847,34 @@ def test_form_with_skip_enforcing_utf8_false
     assert_dom_equal expected, output_buffer
   end
 
+  def test_form_with_default_enforce_utf8_true
+    with_default_enforce_utf8 true do
+      form_with(scope: :post) do |f|
+        concat f.text_field(:title)
+      end
+
+      expected = whole_form("/", skip_enforcing_utf8: false) do
+        "<input name='post[title]' type='text' value='Hello World' id='post_title' />"
+      end
+
+      assert_dom_equal expected, output_buffer
+    end
+  end
+
+  def test_form_with_default_enforce_utf8_false
+    with_default_enforce_utf8 false do
+      form_with(scope: :post) do |f|
+        concat f.text_field(:title)
+      end
+
+      expected = whole_form("/", skip_enforcing_utf8: true) do
+        "<input name='post[title]' type='text' value='Hello World' id='post_title' />"
+      end
+
+      assert_dom_equal expected, output_buffer
+    end
+  end
+
   def test_form_with_without_object
     form_with(scope: :post, id: "create-post") do |f|
       concat f.text_field(:title)
@@ -2291,4 +2347,13 @@ def with_locale(testing_locale = :label)
     ensure
       I18n.locale = old_locale
     end
+
+    def with_default_enforce_utf8(value)
+      old_value = ActionView::Helpers::FormTagHelper.default_enforce_utf8
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = value
+
+      yield
+    ensure
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = old_value
+    end
 end

actionview/test/template/form_helper_test.rb

@@ -1995,6 +1995,34 @@ def test_form_for_enforce_utf8_false
     assert_dom_equal expected, output_buffer
   end
 
+  def test_form_for_default_enforce_utf8_false
+    with_default_enforce_utf8 false do
+      form_for(:post) do |f|
+        concat f.text_field(:title)
+      end
+
+      expected = whole_form("/", nil, nil, enforce_utf8: false) do
+        "<input name='post[title]' type='text' id='post_title' value='Hello World' />"
+      end
+
+      assert_dom_equal expected, output_buffer
+    end
+  end
+
+  def test_form_for_default_enforce_utf8_true
+    with_default_enforce_utf8 true do
+      form_for(:post) do |f|
+        concat f.text_field(:title)
+      end
+
+      expected = whole_form("/", nil, nil, enforce_utf8: true) do
+        "<input name='post[title]' type='text' id='post_title' value='Hello World' />"
+      end
+
+      assert_dom_equal expected, output_buffer
+    end
+  end
+
   def test_form_for_with_remote_in_html
     form_for(@post, url: "/", html: { remote: true, id: "create-post", method: :patch }) do |f|
       concat f.text_field(:title)
@@ -3569,4 +3597,13 @@ def with_locale(testing_locale = :label)
     ensure
       I18n.locale = old_locale
     end
+
+    def with_default_enforce_utf8(value)
+      old_value = ActionView::Helpers::FormTagHelper.default_enforce_utf8
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = value
+
+      yield
+    ensure
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = old_value
+    end
 end

actionview/test/template/form_tag_helper_test.rb

@@ -152,6 +152,24 @@ def test_form_tag_enforce_utf8_false
     assert_predicate actual, :html_safe?
   end
 
+  def test_form_tag_default_enforce_utf8_false
+    with_default_enforce_utf8 false do
+      actual = form_tag({})
+      expected = whole_form("http://www.example.com", enforce_utf8: false)
+      assert_dom_equal expected, actual
+      assert_predicate actual, :html_safe?
+    end
+  end
+
+  def test_form_tag_default_enforce_utf8_true
+    with_default_enforce_utf8 true do
+      actual = form_tag({})
+      expected = whole_form("http://www.example.com", enforce_utf8: true)
+      assert_dom_equal expected, actual
+      assert_predicate actual, :html_safe?
+    end
+  end
+
   def test_form_tag_with_block_in_erb
     output_buffer = render_erb("<%= form_tag('http://www.example.com') do %>Hello world!<% end %>")
 
@@ -782,4 +800,13 @@ def protect_against_forgery?
     def root_elem(rendered_content)
       Nokogiri::HTML::DocumentFragment.parse(rendered_content).children.first # extract from nodeset
     end
+
+    def with_default_enforce_utf8(value)
+      old_value = ActionView::Helpers::FormTagHelper.default_enforce_utf8
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = value
+
+      yield
+    ensure
+      ActionView::Helpers::FormTagHelper.default_enforce_utf8 = old_value
+    end
 end

guides/source/configuring.md

@@ -588,6 +588,8 @@ Defaults to `'signed cookie'`.
 
 * `config.action_view.form_with_generates_ids` determines whether `form_with` generates ids on inputs. This defaults to `true`.
 
+* `config.action_view.default_enforce_utf8` determines whether forms are generated with a hidden tag that forces older versions of Internet Explorer to submit forms encoded in UTF-8. This defaults to `false`.
+
 ### Configuring Action Mailer
 
 There are a number of settings available on `config.action_mailer`:

railties/lib/rails/application/configuration.rb

@@ -117,6 +117,9 @@ def load_defaults(target_version)
         when "6.0"
           load_defaults "5.2"
 
+          if respond_to?(:action_view)
+            action_view.default_enforce_utf8 = false
+          end
         else
           raise "Unknown version #{target_version.to_s.inspect}"
         end

railties/lib/rails/generators/rails/app/app_generator.rb

@@ -459,7 +459,7 @@ def delete_api_initializers
 
       def delete_new_framework_defaults
         unless options[:update]
-          remove_file "config/initializers/new_framework_defaults_5_2.rb"
+          remove_file "config/initializers/new_framework_defaults_6_0.rb"
         end
       end
 

railties/lib/rails/generators/rails/app/templates/config/initializers/new_framework_defaults_5_2.rb.tt

-# Be sure to restart your server when you modify this file.
-#
-# This file contains migration options to ease your Rails 5.2 upgrade.
-#
-# Once upgraded flip defaults one by one to migrate to the new default.
-#
-# Read the Guide for Upgrading Ruby on Rails for more info on each option.
-
-# Make Active Record use stable #cache_key alongside new #cache_version method.
-# This is needed for recyclable cache keys.
-# Rails.application.config.active_record.cache_versioning = true
-
-# Use AES-256-GCM authenticated encryption for encrypted cookies.
-# Existing cookies will be converted on read then written with the new scheme.
-# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true
-
-# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages
-# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true.
-# Rails.application.config.active_support.use_authenticated_message_encryption = true
-
-# Add default protection from forgery to ActionController::Base instead of in
-# ApplicationController.
-# Rails.application.config.action_controller.default_protect_from_forgery = true
-
-# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and
-# 'f' after migrating old data.
-# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
-
-# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header.
-# Rails.application.config.active_support.use_sha1_digests = true

railties/lib/rails/generators/rails/app/templates/config/initializers/new_framework_defaults_6_0.rb.tt

+# Be sure to restart your server when you modify this file.
+#
+# This file contains migration options to ease your Rails 6.0 upgrade.
+#
+# Once upgraded flip defaults one by one to migrate to the new default.
+#
+# Read the Guide for Upgrading Ruby on Rails for more info on each option.
+
+# Don't force requests from old versions of IE to be UTF-8 encoded
+# Rails.application.config.action_controller.default_enforce_utf8 = false

railties/test/application/configuration_test.rb

@@ -1766,7 +1766,7 @@ class Post < ActiveRecord::Base
     test "represent_boolean_as_integer should be able to set via config.active_record.sqlite3.represent_boolean_as_integer" do
       remove_from_config '.*config\.load_defaults.*\n'
 
-      app_file "config/initializers/new_framework_defaults_5_2.rb", <<-RUBY
+      app_file "config/initializers/new_framework_defaults_6_0.rb", <<-RUBY
         Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
       RUBY
 
@@ -1905,8 +1905,8 @@ def index
     test "ActiveSupport::MessageEncryptor.use_authenticated_message_encryption can be configured via config.active_support.use_authenticated_message_encryption" do
       remove_from_config '.*config\.load_defaults.*\n'
 
-      app_file "config/initializers/new_framework_defaults_5_2.rb", <<-RUBY
-      Rails.application.config.active_support.use_authenticated_message_encryption = true
+      app_file "config/initializers/new_framework_defaults_6_0.rb", <<-RUBY
+        Rails.application.config.active_support.use_authenticated_message_encryption = true
       RUBY
 
       app "development"
@@ -1931,8 +1931,8 @@ def index
     test "ActiveSupport::Digest.hash_digest_class can be configured via config.active_support.use_sha1_digests" do
       remove_from_config '.*config\.load_defaults.*\n'
 
-      app_file "config/initializers/new_framework_defaults_5_2.rb", <<-RUBY
-      Rails.application.config.active_support.use_sha1_digests = true
+      app_file "config/initializers/new_framework_defaults_6_0.rb", <<-RUBY
+        Rails.application.config.active_support.use_sha1_digests = true
       RUBY
 
       app "development"
@@ -1952,6 +1952,32 @@ class ::DummySerializer < ActiveJob::Serializers::ObjectSerializer; end
       assert_includes ActiveJob::Serializers.serializers, DummySerializer
     end
 
+    test "ActionView::Helpers::FormTagHelper.default_enforce_utf8 is false by default" do
+      app "development"
+      assert_equal false, ActionView::Helpers::FormTagHelper.default_enforce_utf8
+    end
+
+    test "ActionView::Helpers::FormTagHelper.default_enforce_utf8 is true in an upgraded app" do
+      remove_from_config '.*config\.load_defaults.*\n'
+      add_to_config 'config.load_defaults "5.2"'
+
+      app "development"
+
+      assert_equal true, ActionView::Helpers::FormTagHelper.default_enforce_utf8
+    end
+
+    test "ActionView::Helpers::FormTagHelper.default_enforce_utf8 can be configured via config.action_view.default_enforce_utf8" do
+      remove_from_config '.*config\.load_defaults.*\n'
+
+      app_file "config/initializers/new_framework_defaults_6_0.rb", <<-RUBY
+        Rails.application.config.action_view.default_enforce_utf8 = true
+      RUBY
+
+      app "development"
+
+      assert_equal true, ActionView::Helpers::FormTagHelper.default_enforce_utf8
+    end
+
     private
       def force_lazy_load_hooks
         yield # Tasty clarifying sugar, homie! We only need to reference a constant to load it.

railties/test/generators/app_generator_test.rb

@@ -211,7 +211,7 @@ def test_new_application_not_include_api_initializers
   end
 
   def test_new_application_doesnt_need_defaults
-    assert_no_file "config/initializers/new_framework_defaults_5_2.rb"
+    assert_no_file "config/initializers/new_framework_defaults_6_0.rb"
   end
 
   def test_new_application_load_defaults
@@ -259,14 +259,14 @@ def test_app_update_create_new_framework_defaults
     app_root = File.join(destination_root, "myapp")
     run_generator [app_root]
 
-    assert_no_file "#{app_root}/config/initializers/new_framework_defaults_5_2.rb"
+    assert_no_file "#{app_root}/config/initializers/new_framework_defaults_6_0.rb"
 
     stub_rails_application(app_root) do
       generator = Rails::Generators::AppGenerator.new ["rails"], { update: true }, { destination_root: app_root, shell: @shell }
       generator.send(:app_const)
       quietly { generator.send(:update_config_files) }
 
-      assert_file "#{app_root}/config/initializers/new_framework_defaults_5_2.rb"
+      assert_file "#{app_root}/config/initializers/new_framework_defaults_6_0.rb"
     end
   end