diff --git a/activemodel/lib/active_model/attribute_set.rb b/activemodel/lib/active_model/attribute_set.rb
index 4d86e88d5b7d7796a1881a9195f395f0b846891c..6ee3623699cb52db392886d2a9d77e55e8c27a38 100644
--- a/activemodel/lib/active_model/attribute_set.rb
+++ b/activemodel/lib/active_model/attribute_set.rb
@@ -13,11 +13,11 @@ def initialize(attributes)
     end
 
     def [](name)
-      attributes[name] || Attribute.null(name)
+      @attributes[name] || default_attribute(name)
     end
 
     def []=(name, value)
-      attributes[name] = value
+      @attributes[name] = value
     end
 
     def values_before_type_cast
@@ -25,9 +25,9 @@ def values_before_type_cast
     end
 
     def to_hash
-      initialized_attributes.transform_values(&:value)
+      keys.index_with { |name| self[name].value }
     end
-    alias_method :to_h, :to_hash
+    alias :to_h :to_hash
 
     def key?(name)
       attributes.key?(name) && self[name].initialized?
@@ -42,38 +42,36 @@ def fetch_value(name, &block)
     end
 
     def write_from_database(name, value)
-      attributes[name] = self[name].with_value_from_database(value)
+      @attributes[name] = self[name].with_value_from_database(value)
     end
 
     def write_from_user(name, value)
       raise FrozenError, "can't modify frozen attributes" if frozen?
-      attributes[name] = self[name].with_value_from_user(value)
+      @attributes[name] = self[name].with_value_from_user(value)
       value
     end
 
     def write_cast_value(name, value)
-      attributes[name] = self[name].with_cast_value(value)
+      @attributes[name] = self[name].with_cast_value(value)
       value
     end
 
     def freeze
-      @attributes.freeze
+      attributes.freeze
       super
     end
 
     def deep_dup
-      self.class.allocate.tap do |copy|
-        copy.instance_variable_set(:@attributes, attributes.deep_dup)
-      end
+      AttributeSet.new(attributes.deep_dup)
     end
 
     def initialize_dup(_)
-      @attributes = attributes.dup
+      @attributes = @attributes.dup
       super
     end
 
     def initialize_clone(_)
-      @attributes = attributes.clone
+      @attributes = @attributes.clone
       super
     end
 
@@ -84,7 +82,7 @@ def reset(key)
     end
 
     def accessed
-      attributes.select { |_, attr| attr.has_been_read? }.keys
+      attributes.each_key.select { |name| self[name].has_been_read? }
     end
 
     def map(&block)
@@ -100,8 +98,8 @@ def ==(other)
       attr_reader :attributes
 
     private
-      def initialized_attributes
-        attributes.select { |_, attr| attr.initialized? }
+      def default_attribute(name)
+        Attribute.null(name)
       end
   end
 end
diff --git a/activemodel/lib/active_model/attribute_set/builder.rb b/activemodel/lib/active_model/attribute_set/builder.rb
index 57df36d24a23d8b9dbfd425567eac8650b9acbd0..a6559ac02d94b6605bf3ca88915bdd765d081c32 100644
--- a/activemodel/lib/active_model/attribute_set/builder.rb
+++ b/activemodel/lib/active_model/attribute_set/builder.rb
@@ -13,45 +13,33 @@ def initialize(types, default_attributes = {})
       end
 
       def build_from_database(values = {}, additional_types = {})
-        attributes = LazyAttributeHash.new(types, values, additional_types, default_attributes)
-        LazyAttributeSet.new(attributes)
+        LazyAttributeSet.new(values, types, additional_types, default_attributes)
       end
     end
   end
 
   class LazyAttributeSet < AttributeSet # :nodoc:
-    def fetch_value(name, &block)
-      attributes.fetch_value(name, &block)
-    end
-  end
-
-  class LazyAttributeHash # :nodoc:
-    delegate :transform_values, :each_key, :each_value, :fetch, :except, to: :materialize
-
-    def initialize(types, values, additional_types, default_attributes, delegate_hash = {})
-      @types = types
+    def initialize(values, types, additional_types, default_attributes, attributes = {})
+      super(attributes)
       @values = values
+      @types = types
       @additional_types = additional_types
       @default_attributes = default_attributes
-      @delegate_hash = delegate_hash
       @casted_values = {}
       @materialized = false
     end
 
-    def key?(key)
-      delegate_hash.key?(key) || values.key?(key) || types.key?(key)
+    def key?(name)
+      (values.key?(name) || types.key?(name) || @attributes.key?(name)) && self[name].initialized?
     end
 
-    def [](key)
-      delegate_hash[key] || assign_default_value(key)
-    end
-
-    def []=(key, value)
-      delegate_hash[key] = value
+    def keys
+      keys = values.keys | types.keys | @attributes.keys
+      keys.keep_if { |name| self[name].initialized? }
     end
 
     def fetch_value(name, &block)
-      if attr = delegate_hash[name]
+      if attr = @attributes[name]
         return attr.value(&block)
       end
 
@@ -63,12 +51,70 @@ def fetch_value(name, &block)
           type = additional_types.fetch(name, types[name])
           @casted_values[name] = type.deserialize(value)
         else
-          attr = assign_default_value(name, value_present, value) || Attribute.null(name)
+          attr = default_attribute(name, value_present, value)
           attr.value(&block)
         end
       end
     end
 
+    protected
+      def attributes
+        unless @materialized
+          values.each_key { |key| self[key] }
+          types.each_key { |key| self[key] }
+          @materialized = true
+        end
+        @attributes
+      end
+
+    private
+      attr_reader :values, :types, :additional_types, :default_attributes
+
+      def default_attribute(
+        name,
+        value_present = true,
+        value = values.fetch(name) { value_present = false }
+      )
+        type = additional_types.fetch(name, types[name])
+
+        if value_present
+          @attributes[name] = Attribute.from_database(name, value, type, @casted_values[name])
+        elsif types.key?(name)
+          if attr = default_attributes[name]
+            @attributes[name] = attr.dup
+          else
+            @attributes[name] = Attribute.uninitialized(name, type)
+          end
+        else
+          Attribute.null(name)
+        end
+      end
+  end
+
+  class LazyAttributeHash # :nodoc:
+    delegate :transform_values, :each_value, :fetch, :except, to: :materialize
+
+    def initialize(types, values, additional_types, default_attributes, delegate_hash = {})
+      @types = types
+      @values = values
+      @additional_types = additional_types
+      @materialized = false
+      @delegate_hash = delegate_hash
+      @default_attributes = default_attributes
+    end
+
+    def key?(key)
+      delegate_hash.key?(key) || values.key?(key) || types.key?(key)
+    end
+
+    def [](key)
+      delegate_hash[key] || assign_default_value(key)
+    end
+
+    def []=(key, value)
+      delegate_hash[key] = value
+    end
+
     def deep_dup
       dup.tap do |copy|
         copy.instance_variable_set(:@delegate_hash, delegate_hash.transform_values(&:dup))
@@ -80,14 +126,9 @@ def initialize_dup(_)
       super
     end
 
-    def select
+    def each_key(&block)
       keys = types.keys | values.keys | delegate_hash.keys
-      keys.each_with_object({}) do |key, hash|
-        attribute = self[key]
-        if yield(key, attribute)
-          hash[key] = attribute
-        end
-      end
+      keys.each(&block)
     end
 
     def ==(other)
@@ -130,15 +171,13 @@ def materialize
     private
       attr_reader :types, :values, :additional_types, :delegate_hash, :default_attributes
 
-      def assign_default_value(
-        name,
-        value_present = true,
-        value = values.fetch(name) { value_present = false }
-      )
+      def assign_default_value(name)
         type = additional_types.fetch(name, types[name])
+        value_present = true
+        value = values.fetch(name) { value_present = false }
 
         if value_present
-          delegate_hash[name] = Attribute.from_database(name, value, type, @casted_values[name])
+          delegate_hash[name] = Attribute.from_database(name, value, type)
         elsif types.key?(name)
           attr = default_attributes[name]
           if attr
diff --git a/activemodel/test/cases/attribute_set_test.rb b/activemodel/test/cases/attribute_set_test.rb
index 6494b9ef406f83ee402a40868a4c0a5f5b732c64..3ddfe6f8ea94fcc994bfbc57b2ed9162fb222d25 100644
--- a/activemodel/test/cases/attribute_set_test.rb
+++ b/activemodel/test/cases/attribute_set_test.rb
@@ -219,6 +219,12 @@ def assert_valid_value(*)
 
     test "marshalling dump/load legacy materialized attribute hash" do
       builder = AttributeSet::Builder.new(foo: Type::String.new)
+
+      def builder.build_from_database(values = {}, additional_types = {})
+        attributes = LazyAttributeHash.new(types, values, additional_types, default_attributes)
+        AttributeSet.new(attributes)
+      end
+
       attributes = builder.build_from_database(foo: "1")
 
       attributes.instance_variable_get(:@attributes).instance_eval do