diff --git a/activesupport/lib/active_support/cache/memory_store.rb b/activesupport/lib/active_support/cache/memory_store.rb
index 469758f435c74d18f1512d0a0fd9ec252b4fff90..274b7394b142af35043514a6cc0fd9e14ca3c995 100644
--- a/activesupport/lib/active_support/cache/memory_store.rb
+++ b/activesupport/lib/active_support/cache/memory_store.rb
@@ -22,7 +22,6 @@ def initialize(options = nil)
         options ||= {}
         super(options)
         @data = {}
-        @key_access = {}
         @max_size = options[:size] || 32.megabytes
         @max_prune_time = options[:max_prune_time] || 2
         @cache_size = 0
@@ -39,7 +38,6 @@ def self.supports_cache_versioning?
       def clear(options = nil)
         synchronize do
           @data.clear
-          @key_access.clear
           @cache_size = 0
         end
       end
@@ -65,7 +63,7 @@ def prune(target_size, max_time = nil)
           start_time = Concurrent.monotonic_time
           cleanup
           instrument(:prune, target_size, from: @cache_size) do
-            keys = synchronize { @key_access.keys.sort { |a, b| @key_access[a].to_f <=> @key_access[b].to_f } }
+            keys = synchronize { @data.keys }
             keys.each do |key|
               delete_entry(key, **options)
               return if @cache_size <= target_size || (max_time && Concurrent.monotonic_time - start_time > max_time)
@@ -121,30 +119,29 @@ def cached_size(key, entry)
         end
 
         def read_entry(key, **options)
-          entry = @data[key]
+          entry = nil
           synchronize do
+            entry = @data.delete(key)
             if entry
+              @data[key] = entry
               entry = entry.dup
-              entry.dup_value!
-              @key_access[key] = Time.now.to_f
-            else
-              @key_access.delete(key)
             end
           end
+          entry&.dup_value!
           entry
         end
 
         def write_entry(key, entry, **options)
           entry.dup_value!
           synchronize do
-            old_entry = @data[key]
-            return false if @data.key?(key) && options[:unless_exist]
+            return false if options[:unless_exist] && @data.key?(key)
+
+            old_entry = @data.delete(key)
             if old_entry
               @cache_size -= (old_entry.size - entry.size)
             else
               @cache_size += cached_size(key, entry)
             end
-            @key_access[key] = Time.now.to_f
             @data[key] = entry
             prune(@max_size * 0.75, @max_prune_time) if @cache_size > @max_size
             true
@@ -153,7 +150,6 @@ def write_entry(key, entry, **options)
 
         def delete_entry(key, **options)
           synchronize do
-            @key_access.delete(key)
             entry = @data.delete(key)
             @cache_size -= cached_size(key, entry) if entry
             !!entry