diff --git a/activerecord/lib/active_record/associations/preloader/association.rb b/activerecord/lib/active_record/associations/preloader/association.rb
index 5c68e2e3be0023bc59626be8b1a775ed3b9c96b3..fb620b0f7b4bbf32154d7ee1f30d431e774fe0d0 100644
--- a/activerecord/lib/active_record/associations/preloader/association.rb
+++ b/activerecord/lib/active_record/associations/preloader/association.rb
@@ -27,39 +27,41 @@ def run
         end
 
         def records_by_owner
-          # owners can be duplicated when a relation has a collection association join
-          # #compare_by_identity makes such owners different hash keys
-          @records_by_owner ||= preloaded_records.each_with_object({}.compare_by_identity) do |record, result|
-            owners_by_key[convert_key(record[association_key_name])].each do |owner|
-              (result[owner] ||= []) << record
-            end
-          end
+          load_records unless defined?(@records_by_owner)
+
+          @records_by_owner
         end
 
         def preloaded_records
-          return @preloaded_records if defined?(@preloaded_records)
+          load_records unless defined?(@preloaded_records)
+
+          @preloaded_records
+        end
+
+        private
+          attr_reader :owners, :reflection, :preload_scope, :model, :klass
 
-          raw_records = owner_keys.empty? ? [] : records_for(owner_keys)
-          seen_records_by_owner = {}.compare_by_identity
+          def load_records
+            # owners can be duplicated when a relation has a collection association join
+            # #compare_by_identity makes such owners different hash keys
+            @records_by_owner = {}.compare_by_identity
+            raw_records = owner_keys.empty? ? [] : records_for(owner_keys)
 
-          @preloaded_records = raw_records.select do |record|
-            assignments = []
+            @preloaded_records = raw_records.select do |record|
+              assignments = []
 
-            owners_by_key[convert_key(record[association_key_name])].each do |owner|
-              entries = (seen_records_by_owner[owner] ||= [])
+              owners_by_key[convert_key(record[association_key_name])].each do |owner|
+                entries = (@records_by_owner[owner] ||= [])
 
-              if reflection.collection? || entries.empty?
-                entries << record
-                assignments << record
+                if reflection.collection? || entries.empty?
+                  entries << record
+                  assignments << record
+                end
               end
-            end
 
-            !assignments.empty?
+              !assignments.empty?
+            end
           end
-        end
-
-        private
-          attr_reader :owners, :reflection, :preload_scope, :model, :klass
 
           # The name of the key on the associated records
           def association_key_name