diff --git a/CHANGELOG.md b/CHANGELOG.md
index d306e1a7d607f4c89abddc1eaa439ced90e80ec0..adaf650a179751e4f8b23794856db31ba0647280 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#2890 Fix the index size caculation in cache
 -   \#2952 Fix the result merging of IVF_PQ IP
 -   \#2975 Fix config UT failed
+-   \#3012 If the cache is too small, queries using multiple GPUs will cause to crash
 
 ## Feature
 
diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp
index 4970b094ede13a1a5640ac955517b85e2961cba4..cbc7aa9194d00fd0c7401ea4daa2d7c59cb8ee77 100644
--- a/core/src/db/engine/ExecutionEngineImpl.cpp
+++ b/core/src/db/engine/ExecutionEngineImpl.cpp
@@ -588,6 +588,11 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
             bool gpu_cache_enable = false;
             STATUS_CHECK(server::Config::GetInstance().GetGpuResourceConfigCacheEnable(gpu_cache_enable));
 
+            /* CopyCpuToGpu() is an asynchronous method.
+             * It should be make sure that the CPU index is always valid.
+             * Therefore, we reserve its shared pointer.
+             */
+            index_reserve_ = index_;
             if (gpu_cache_enable) {
                 gpu_cache_mgr->Reserve(index_->Size());
                 index_ = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h
index b4e3de73c729944af632724b8c39b356b6f9eb11..30eba674cdda8c8230ea75332fc632cac893551d 100644
--- a/core/src/db/engine/ExecutionEngineImpl.h
+++ b/core/src/db/engine/ExecutionEngineImpl.h
@@ -123,6 +123,9 @@ class ExecutionEngineImpl : public ExecutionEngine {
 
  protected:
     knowhere::VecIndexPtr index_ = nullptr;
+#ifdef MILVUS_GPU_VERSION
+    knowhere::VecIndexPtr index_reserve_ = nullptr;  // reserve the cpu index before copying it to gpu
+#endif
     std::string location_;
     int64_t dim_;
     EngineType index_type_;