[CustomDevice] fix bug in custom allocator (#56977)

paddle/fluid/memory/allocation/custom_allocator.cc

@@ -29,7 +29,8 @@ void CustomAllocator::FreeImpl(phi::Allocation* allocation) {
       platform::errors::PermissionDenied("CustomDevice memory is "
                                          "freed in incorrect device. "
                                          "This may be a bug"));
-
+  phi::DeviceManager::GetDeviceWithPlace(place_)->MemoryDeallocate(
+      allocation->ptr(), allocation->size());
   delete allocation;
 }