[XPU] fix l3 malloc (#54114)

paddle/fluid/inference/api/analysis_predictor.cc

@@ -2180,7 +2180,7 @@ bool AnalysisPredictor::ExpRunWithRuntimeConfig(void *config) {
           "l3_autotune_size(%zu) should be less than or equal to l3_size(%zu).",
           l3_autotune_size,
           l3_size));
-  dev_ctx->SetL3Info(l3_size, l3_ptr, l3_autotune_size);
+  dev_ctx->SetL3Info(l3_size, l3_ptr, l3_autotune_size, place_);
 
   bool ret = ZeroCopyRun();
   dev_ctx->L3CacheAutotune();

paddle/fluid/inference/api/infer_context.cc

@@ -77,7 +77,9 @@ void* InferXPUContext::Alloc(phi::TensorBase* tensor,
 
 void InferXPUContext::SetL3Info(size_t l3_size,
                                 void* l3_ptr,
-                                size_t l3_autotune_size) {
+                                size_t l3_autotune_size,
+                                const phi::Place& place) {
+  phi::backends::xpu::XPUDeviceGuard guard(place.GetDeviceId());
   if (l3_ptr == nullptr) {
     if (l3_size_ != l3_size) {
       if (l3_owned_) {

paddle/fluid/inference/api/infer_context.h

@@ -60,7 +60,10 @@ class InferXPUContext : public phi::XPUContext {
               bool pinned = false,
               bool fake_alloc = false) const override;
 
-  void SetL3Info(size_t l3_size, void* l3_ptr, size_t l3_autotune_size);
+  void SetL3Info(size_t l3_size,
+                 void* l3_ptr,
+                 size_t l3_autotune_size,
+                 const phi::Place& place);
 
   void L3CacheAutotune();