Use the query cache when connection is already connected

Fixes #32021.

activerecord/lib/active_record/query_cache.rb

@@ -7,20 +7,20 @@ module ClassMethods
       # Enable the query cache within the block if Active Record is configured.
       # If it's not, it will execute the given block.
       def cache(&block)
-        if configurations.empty?
-          yield
-        else
+        if connected? || !configurations.empty?
           connection.cache(&block)
+        else
+          yield
         end
       end
 
       # Disable the query cache within the block if Active Record is configured.
       # If it's not, it will execute the given block.
       def uncached(&block)
-        if configurations.empty?
-          yield
-        else
+        if connected? || !configurations.empty?
           connection.uncached(&block)
+        else
+          yield
         end
       end
     end

activerecord/test/cases/query_cache_test.rb

@@ -320,6 +320,17 @@ def test_cache_is_ignored_for_locked_relations
     end
   end
 
+  def test_cache_is_available_when_connection_is_connected
+    conf = ActiveRecord::Base.configurations
+
+    ActiveRecord::Base.configurations = {}
+    Task.cache do
+      assert_queries(1) { Task.find(1); Task.find(1) }
+    end
+  ensure
+    ActiveRecord::Base.configurations = conf
+  end
+
   def test_cache_is_available_when_using_a_not_connected_connection
     skip "In-Memory DB can't test for using a not connected connection" if in_memory_db?
     with_temporary_connection_pool do